/*
 * Licensed to the Apache Software Foundation (ASF) under one
 * or more contributor license agreements.  See the NOTICE file
 * distributed with this work for additional information
 * regarding copyright ownership.  The ASF licenses this file
 * to you under the Apache License, Version 2.0 (the
 * "License"); you may not use this file except in compliance
 * with the License.  You may obtain a copy of the License at
 *
 *   http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing,
 * software distributed under the License is distributed on an
 * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
 * KIND, either express or implied.  See the License for the
 * specific language governing permissions and limitations
 * under the License.
 */

/*!
 * \brief Infer TensorCore metadata from tensor intrinsic.
 * \file tensorcore_fragment.cc
 */
#include <tvm/ffi/cast.h>
#include <tvm/ffi/function.h>
#include <tvm/ffi/reflection/registry.h>
#include <tvm/ir/op.h>
#include <tvm/s_tir/stmt.h>
#include <tvm/s_tir/transform.h>
#include <tvm/tirx/expr.h>
#include <tvm/tirx/stmt_functor.h>

#include <unordered_map>
#include <unordered_set>

#include "../../runtime/thread_storage_scope.h"
#include "../../tirx/transform/ir_utils.h"
#include "storage_access.h"

namespace tvm {
namespace s_tir {
using namespace tvm::tirx;

const VarNode* GetBufferVarFromData(const Expr& data) {
  if (const auto* var = data.as<VarNode>()) {
    return var;
  }
  if (const auto* call = data.as<CallNode>();
      call && call->op.same_as(builtin::buffer_data()) && call->args.size() == 1) {
    return call->args[0].as<VarNode>();
  }
  return nullptr;
}

// Get fragment information from tensor intrinsics
class FragmentGetter : public StmtExprVisitor {
 public:
  void VisitExpr_(const CallNode* op) final {
    StmtExprVisitor::VisitExpr_(op);

    static const Op& tvm_load_matrix_sync_op = Op::Get("tirx.tvm_load_matrix_sync");
    static const Op& tvm_store_matrix_sync_op = Op::Get("tirx.tvm_store_matrix_sync");
    static const Op& tvm_fill_fragment_op = Op::Get("tirx.tvm_fill_fragment");
    if (op->op.same_as(tvm_load_matrix_sync_op) || op->op.same_as(tvm_store_matrix_sync_op)) {
      // Get shape and layout information from load and store intrinsic
      TVM_FFI_ICHECK_EQ(op->args.size(), 8U);
      const VarNode* buffer_var = GetBufferVarFromData(op->args[0]);
      TVM_FFI_ICHECK(buffer_var);
      // Get shape
      const IntImmNode* m = op->args[1].as<IntImmNode>();
      const IntImmNode* n = op->args[2].as<IntImmNode>();
      const IntImmNode* k = op->args[3].as<IntImmNode>();
      const StringImmNode* layout = op->args[7].as<StringImmNode>();
      TVM_FFI_ICHECK(m);
      TVM_FFI_ICHECK(n);
      TVM_FFI_ICHECK(k);
      TVM_FFI_ICHECK(layout);

      std::string scope = GetPtrStorageScope(ffi::GetRef<Var>(buffer_var));
      if (fragments.count(buffer_var)) {
        // check if the fragment has met before
        FragmentInfo info = fragments[buffer_var];
        TVM_FFI_ICHECK_EQ(m->value, info.m);
        TVM_FFI_ICHECK_EQ(n->value, info.n);
        TVM_FFI_ICHECK_EQ(k->value, info.k);
        if (scope == "wmma.matrix_a" || scope == "wmma.matrix_b") {
          TVM_FFI_ICHECK_EQ(layout->value, info.layout);
        }
      } else {
        // store metadata
        FragmentInfo info;
        if (scope == "wmma.matrix_a" || scope == "wmma.matrix_b") {
          info = FragmentInfo(m->value, n->value, k->value, layout->value, scope);
        } else if (scope == "wmma.accumulator") {
          info = FragmentInfo(m->value, n->value, k->value, "", scope);
        }
        fragments[buffer_var] = info;
      }
    } else if (op->op.same_as(tvm_fill_fragment_op)) {
      // Get shape information from fill intrinsic
      TVM_FFI_ICHECK_EQ(op->args.size(), 6U);
      const VarNode* buffer_var = GetBufferVarFromData(op->args[0]);
      TVM_FFI_ICHECK(buffer_var);
      // Get shape
      const IntImmNode* m = op->args[1].as<IntImmNode>();
      const IntImmNode* n = op->args[2].as<IntImmNode>();
      const IntImmNode* k = op->args[3].as<IntImmNode>();
      TVM_FFI_ICHECK(m);
      TVM_FFI_ICHECK(n);
      TVM_FFI_ICHECK(k);

      std::string scope = GetPtrStorageScope(ffi::GetRef<Var>(buffer_var));
      if (fragments.count(buffer_var)) {
        FragmentInfo info = fragments[buffer_var];
        TVM_FFI_ICHECK_EQ(m->value, info.m);
        TVM_FFI_ICHECK_EQ(n->value, info.n);
        TVM_FFI_ICHECK_EQ(k->value, info.k);
      } else {
        // default to row major ordering
        FragmentInfo info(m->value, n->value, k->value, "row_major", scope);
        fragments[buffer_var] = info;
      }
    }
  }

  // Get memory scope
  void VisitStmt_(const AttrStmtNode* op) final { StmtExprVisitor::VisitStmt_(op); }

  // Fragment metadata for all fragments
  std::unordered_map<const VarNode*, FragmentInfo> fragments;
};

}  // namespace s_tir

namespace tirx {
std::unordered_map<const VarNode*, FragmentInfo> GetTensorCoreFragmentInfo(const Stmt& stmt) {
  s_tir::FragmentGetter getter;
  getter(stmt);
  return std::move(getter.fragments);
}
}  // namespace tirx

namespace s_tir {

// Check shape of fragment making sure it is a valid shape for tvm_mma_sync
class FragmentChecker : public StmtExprVisitor {
 public:
  explicit FragmentChecker(const FragmentGetter& getter) : fragment_getter(getter) {}

  void VisitExpr_(const CallNode* op) final {
    StmtExprVisitor::VisitExpr_(op);
    // Check shape when calling tvm_mma_sync
    static const Op& tvm_mma_sync_op = Op::Get("tirx.tvm_mma_sync");
    static const Op& tvm_bmma_sync_op = Op::Get("tirx.tvm_bmma_sync");
    if (op->op.same_as(tvm_mma_sync_op) || op->op.same_as(tvm_bmma_sync_op)) {
      TVM_FFI_ICHECK_EQ(op->args.size(), 8U);
      const VarNode* buffer_var_d = GetBufferVarFromData(op->args[0]);
      const VarNode* buffer_var_a = GetBufferVarFromData(op->args[2]);
      const VarNode* buffer_var_b = GetBufferVarFromData(op->args[4]);
      const VarNode* buffer_var_c = GetBufferVarFromData(op->args[6]);
      TVM_FFI_ICHECK(buffer_var_d);
      TVM_FFI_ICHECK(buffer_var_a);
      TVM_FFI_ICHECK(buffer_var_b);
      TVM_FFI_ICHECK(buffer_var_c);

      // Check all fragment A, B, C and D have the same shape
      TVM_FFI_ICHECK(CheckShape(buffer_var_d, buffer_var_a));
      TVM_FFI_ICHECK(CheckShape(buffer_var_d, buffer_var_b));
      TVM_FFI_ICHECK(CheckShape(buffer_var_d, buffer_var_c));
    }
  }

 private:
  // A tool for checking shapes of two fragments
  bool CheckShape(const VarNode* buffer1, const VarNode* buffer2) {
    TVM_FFI_ICHECK(fragment_getter.fragments.count(buffer1))
        << "Tensorecore fragment " << buffer1->name
        << " must be filled (with tvm_fill_fragment) or loaded (with tvm_load_matrix_sync) before "
           "use.";
    TVM_FFI_ICHECK(fragment_getter.fragments.count(buffer2))
        << "Tensorecore fragment " << buffer2->name
        << " must be filled (with tvm_fill_fragment) or loaded (with tvm_load_matrix_sync) before "
           "use.";
    FragmentInfo info1 = fragment_getter.fragments.at(buffer1);
    FragmentInfo info2 = fragment_getter.fragments.at(buffer2);
    return info1.m == info2.m && info1.n == info2.n && info1.k == info2.k;
  }
  // Fragment infomation
  const FragmentGetter& fragment_getter;
};

// Store the metadata into attributes
class InferFragmenter : public StmtMutator {
 public:
  explicit InferFragmenter(const FragmentGetter& getter) : fragment_getter(getter) {}

  Stmt VisitStmt_(const AllocBufferNode* op) final {
    Stmt stmt = StmtMutator::VisitStmt_(op);
    const VarNode* buffer = op->buffer.get();
    if (fragment_getter.fragments.count(buffer)) {
      FragmentInfo info = fragment_getter.fragments.at(buffer);

      std::string shape =
          std::to_string(info.m) + ", " + std::to_string(info.n) + ", " + std::to_string(info.k);
      PrimExpr shape_expr = StringImm(shape);
      Stmt shape_attr = AttrStmt(op->buffer.var(), s_tir::attr::fragment_shape, shape_expr, stmt);
      if (info.layout != "") {
        Stmt layout_attr = AttrStmt(op->buffer.var(), s_tir::attr::fragment_layout,
                                    StringImm(info.layout), shape_attr);
        return layout_attr;
      } else {
        return shape_attr;
      }
    }
    return stmt;
  }

 private:
  // Fragment infomation
  const FragmentGetter& fragment_getter;
};

Stmt InferFragment(Stmt stmt) {
  FragmentGetter getter;
  getter(stmt);
  FragmentChecker checker(getter);
  checker(stmt);
  stmt = InferFragmenter(getter)(std::move(stmt));
  return stmt;
}

namespace transform {

Pass InferFragment() {
  auto pass_func = [](PrimFunc f, IRModule m, PassContext ctx) {
    auto* n = f.CopyOnWrite();
    n->body = s_tir::InferFragment(std::move(n->body));
    return f;
  };
  return CreatePrimFuncPass(pass_func, 0, "s_tir.InferFragment", {});
}

TVM_FFI_STATIC_INIT_BLOCK() {
  namespace refl = tvm::ffi::reflection;
  refl::GlobalDef().def("s_tir.transform.InferFragment", static_cast<Pass (*)()>(InferFragment));
}

}  // namespace transform
}  // namespace s_tir
}  // namespace tvm
