Iocgns_DecompositionData.h

Go to the documentation of this file.

/*
 * Copyright(C) 1999-2022, 2024 National Technology & Engineering Solutions
 * of Sandia, LLC (NTESS).  Under the terms of Contract DE-NA0003525 with
 * NTESS, the U.S. Government retains certain rights in this software.
 *
 * See packages/seacas/LICENSE for details
 */
#pragma once
 
#include "iocgns_export.h"
 
#include <cgnsconfig.h>
#if CG_BUILD_PARALLEL
 
#include <string>
#include <vector>
 
#define CG_USE_ROBIN
#if defined CG_USE_STD
#include <unordered_map>
#elif defined CG_USE_HOPSCOTCH
#include <bhopscotch_map.h>
#elif defined CG_USE_ROBIN
#include <robin_map.h>
#endif
 
#include <cstddef>
#include <cstdint>
 
#include "Ioss_CodeTypes.h"
#include "Ioss_Decomposition.h"
#include "Ioss_FaceGenerator.h"
#include "Ioss_Field.h"
#include "Ioss_MeshType.h"
#include "Ioss_PropertyManager.h"
#include "Ioss_StructuredBlock.h"
#include "cgns/Iocgns_StructuredZoneData.h"
 
#include <cgnslib.h>
 
#if 0
#if !defined(NO_PARMETIS_SUPPORT)
#include <parmetis.h>
#endif
#endif
 
#undef MPICPP
#if !defined(NO_ZOLTAN_SUPPORT)
#include <zoltan_cpp.h>
#endif
namespace Ioss {
  class Field;
  template <typename INT> class Decomposition;
} // namespace Ioss
 
namespace Iocgns {
 
  class IOCGNS_EXPORT ZoneData
  {
  public:
    std::string m_name;
    size_t      m_nodeOffset;
    size_t      m_nodeCount;
    size_t      m_elementOffset;
  };
 
  class IOCGNS_EXPORT DecompositionDataBase
  {
  public:
    DecompositionDataBase() = default;
 
    virtual ~DecompositionDataBase();
    virtual void                  decompose_model(int filePtr, Ioss::MeshType mesh_type) = 0;
    IOSS_NODISCARD virtual size_t ioss_node_count() const                                = 0;
    IOSS_NODISCARD virtual size_t ioss_elem_count() const                                = 0;
    IOSS_NODISCARD virtual int    int_size() const                                       = 0;
 
    IOSS_NODISCARD virtual int    spatial_dimension() const = 0;
    IOSS_NODISCARD virtual size_t global_node_count() const = 0;
    IOSS_NODISCARD virtual size_t global_elem_count() const = 0;
 
    IOSS_NODISCARD virtual size_t decomp_node_offset() const = 0;
    IOSS_NODISCARD virtual size_t decomp_node_count() const  = 0;
    IOSS_NODISCARD virtual size_t decomp_elem_offset() const = 0;
    IOSS_NODISCARD virtual size_t decomp_elem_count() const  = 0;
 
    IOSS_NODISCARD virtual std::vector<double> &centroids() = 0;
 
    IOSS_NODISCARD virtual size_t get_commset_node_size() const = 0;
 
    virtual void get_node_coordinates(int filePtr, double *ioss_data,
                                      const Ioss::Field &field) const = 0;
 
    void get_block_connectivity(int filePtr, void *data, int blk_seq, bool raw_ids = false) const;
 
    void get_element_field(int filePtr, int solution_index, int blk_seq, int field_index,
                           double *data) const;
 
    void get_node_field(int filePtr, int step, int field_index, double *data) const;
 
    void get_node_entity_proc_data(void *entity_proc, const Ioss::MapContainer &node_map,
                                   bool do_map) const;
 
    template <typename T>
    void communicate_element_data(T *file_data, T *ioss_data, size_t comp_count) const;
 
    template <typename T>
    void communicate_node_data(T *file_data, T *ioss_data, size_t comp_count) const;
 
    void get_sideset_element_side(int filePtr, const Ioss::SetDecompositionData &sset,
                                  void *data) const;
 
    std::vector<ZoneData>                     m_zones;
    std::vector<Ioss::BlockDecompositionData> m_elementBlocks;
    std::vector<Ioss::SetDecompositionData>   m_sideSets;
    std::vector<Iocgns::StructuredZoneData *> m_structuredZones;
 
    // Maps nodes shared between zones.
    // TODO: Currently each processor has same map; need to figure out how to reduce size
#if defined CG_USE_STD
    using ZoneSharedMap = std::unordered_map<cgsize_t, cgsize_t>;
#elif defined CG_USE_HOPSCOTCH
    //    using ZoneSharedMap = tsl::hopscotch_map<cgsize_t, cgsize_t>;
    using ZoneSharedMap = tsl::bhopscotch_map<cgsize_t, cgsize_t>;
#elif defined CG_USE_ROBIN
    using ZoneSharedMap = tsl::robin_map<cgsize_t, cgsize_t>;
#endif
    ZoneSharedMap m_zoneSharedMap;
  };
 
  template <typename INT> class DecompositionData : public DecompositionDataBase
  {
  public:
    DecompositionData(const Ioss::PropertyManager &props, Ioss_MPI_Comm communicator);
    ~DecompositionData() override = default;
 
    IOSS_NODISCARD int int_size() const override { return sizeof(INT); }
 
    void decompose_model(int filePtr, Ioss::MeshType mesh_type) override;
 
    IOSS_NODISCARD int spatial_dimension() const override
    {
      return m_decomposition.m_spatialDimension;
    }
 
    IOSS_NODISCARD size_t global_node_count() const override
    {
      return m_decomposition.global_node_count();
    }
    IOSS_NODISCARD size_t global_elem_count() const override
    {
      return m_decomposition.global_elem_count();
    }
 
    IOSS_NODISCARD size_t ioss_node_count() const override
    {
      return m_decomposition.ioss_node_count();
    }
    IOSS_NODISCARD size_t ioss_elem_count() const override
    {
      return m_decomposition.ioss_elem_count();
    }
 
    IOSS_NODISCARD size_t decomp_node_offset() const override
    {
      return m_decomposition.file_node_offset();
    }
    IOSS_NODISCARD size_t decomp_node_count() const override
    {
      return m_decomposition.file_node_count();
    }
    IOSS_NODISCARD size_t decomp_elem_offset() const override
    {
      return m_decomposition.file_elem_offset();
    }
    IOSS_NODISCARD size_t decomp_elem_count() const override
    {
      return m_decomposition.file_elem_count();
    }
 
    IOSS_NODISCARD std::vector<double> &centroids() override { return m_decomposition.m_centroids; }
 
    template <typename T>
    void communicate_element_data(T *file_data, T *ioss_data, size_t comp_count) const
    {
      m_decomposition.communicate_element_data(file_data, ioss_data, comp_count);
    }
 
    template <typename T>
    void communicate_set_data(T *file_data, T *ioss_data, const Ioss::SetDecompositionData &set,
                              size_t comp_count) const
    {
      m_decomposition.communicate_set_data(file_data, ioss_data, set, comp_count);
    }
 
    template <typename T>
    void communicate_node_data(T *file_data, T *ioss_data, size_t comp_count) const
    {
      m_decomposition.communicate_node_data(file_data, ioss_data, comp_count);
    }
 
    template <typename U, typename T>
    void communicate_block_data(U *file_data, T *ioss_data,
                                const Ioss::BlockDecompositionData &block, size_t comp_count) const
    {
      m_decomposition.communicate_block_data(file_data, ioss_data, block, comp_count);
    }
 
    void get_block_connectivity(int filePtr, INT *data, int blk_seq, bool raw_ids) const;
 
    void get_element_field(int filePtr, int solution_index, int blk_seq, int field_index,
                           double *data) const;
 
    void get_node_field(int filePtr, int step, int field_offset, double *data) const;
 
    IOSS_NODISCARD size_t get_commset_node_size() const override
    {
      return m_decomposition.m_nodeCommMap.size() / 2;
    }
 
    void get_sideset_element_side(int filePtr, const Ioss::SetDecompositionData &sset,
                                  INT *data) const;
 
  private:
    void decompose_structured(int filePtr);
    void decompose_unstructured(int filePtr);
 
    void get_sideset_data(int filePtr);
    void generate_zone_shared_nodes(int filePtr, INT min_node, INT max_node);
 
    IOSS_NODISCARD bool i_own_node(size_t node)
        const // T/F if node with global index node owned by this processors ioss-decomp.
    {
      return m_decomposition.i_own_node(node);
    }
 
    IOSS_NODISCARD bool i_own_elem(size_t elem)
        const // T/F if node with global index elem owned by this processors ioss-decomp.
    {
      return m_decomposition.i_own_elem(elem);
    }
 
    // global_index is 1-based index into global list of nodes [1..global_node_count]
    // return value is 1-based index into local list of nodes on this
    // processor (ioss-decomposition)
    IOSS_NODISCARD size_t node_global_to_local(size_t global_index) const
    {
      return m_decomposition.node_global_to_local(global_index);
    }
 
    IOSS_NODISCARD size_t elem_global_to_local(size_t global_index) const
    {
      return m_decomposition.elem_global_to_local(global_index);
    }
 
    void build_global_to_local_elem_map()
    {
      return m_decomposition.build_global_to_local_elem_map();
    }
 
    void get_element_block_communication()
    {
      m_decomposition.get_element_block_communication(m_elementBlocks);
    }
 
    void generate_adjacency_list(int filePtr, Ioss::Decomposition<INT> &decomposition);
 
    void calculate_element_centroids(int filePtr, std::vector<double> &centroids);
 
    void get_shared_node_list() { m_decomposition.get_shared_node_list(); }
 
    void get_local_node_list() { m_decomposition.get_local_node_list(); }
 
    void get_file_node_coordinates(int filePtr, int direction, double *ioss_data) const;
    void get_node_coordinates(int filePtr, double *ioss_data,
                              const Ioss::Field &field) const override;
 
    double      m_loadBalanceThreshold{1.4};
    std::string m_lineDecomposition{};
 
    mutable std::map<int, Ioss::FaceUnorderedSet> m_boundaryFaces;
 
  public:
    Ioss::Decomposition<INT> m_decomposition;
  };
 
} // namespace Iocgns
#endif