diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index ad307b19a499644507163e402cf992bff8b70cf2..c8015fac03317097fca259aa4b3ea26842f05c56 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -19,7 +19,7 @@ variables:
#
# multi-step pipeline for each commit
#
-# Mote: "WIP:" merge request, non-WIP merge requests and "Ready for Code Review" MR are all
+# Mote: "Draft:" merge request, non-Draft merge requests and "Ready for Code Review" MR are all
# handled explicitly
#
stages:
@@ -55,8 +55,6 @@ check-copyrights:
##########################################################
check-clang-format:
image: corsika/devel:u-18.04
- before_script:
- - apt-get update && apt-get install -y -qq clang-format
stage: quality
tags:
- corsika
@@ -221,7 +219,7 @@ test-clang-8:
-####### BUILD-TEST (all builds <= WIP, default) ##############
+####### BUILD-TEST (all builds <= Draft, default) ##############
##########################################################
# the generic build_test template job
@@ -236,7 +234,7 @@ test-clang-8:
- set -o pipefail
- ctest -j4 -VV | gzip -v -9 > test.log.gz
rules:
- - if: '$CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_TITLE =~ /^WIP:/'
+ - if: '$CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_TITLE =~ /^Draft:/'
allow_failure: false
- if: $CI_MERGE_REQUEST_ID
when: manual
@@ -330,7 +328,7 @@ example-clang-8:
-####### BUILD-TEST-EXAMPLE (only non-WIP) ##############
+####### BUILD-TEST-EXAMPLE (only non-Draft) ##############
##########################################################
# generic example template job
@@ -346,7 +344,7 @@ example-clang-8:
- ctest -j4 -VV | gzip -v -9 > test.log.gz
- make -j4 run_examples | gzip -v -9 > examples.log.gz
rules:
- - if: '$CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_TITLE =~ /^WIP:/'
+ - if: '$CI_MERGE_REQUEST_ID && $CI_MERGE_REQUEST_TITLE =~ /^Draft:/'
when: manual
allow_failure: true
- if: $CI_COMMIT_BRANCH
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 97e561cf730d17c773e5a467d75a0b40aaaaabc8..82a2b8115df86cde009929b01d5c9b64230d89e1 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -93,6 +93,9 @@ endif ()
if (CMAKE_BUILD_TYPE STREQUAL Coverage)
find_package (Perl REQUIRED)
+ # compile coverage under -O2 to remove unused functions
+ add_compile_options ("-O2")
+
set (GCOV gcov CACHE STRING "gcov executable" FORCE)
set (LCOV_BIN_DIR "${PROJECT_SOURCE_DIR}/ThirdParty/lcov/bin")
# collect coverage data
diff --git a/Framework/Utilities/CMakeLists.txt b/Framework/Utilities/CMakeLists.txt
index b5796bfd535759eae184faed36693902527255df..5618ab4a8cf35be1e36630eac8d3f8d0ab627e95 100644
--- a/Framework/Utilities/CMakeLists.txt
+++ b/Framework/Utilities/CMakeLists.txt
@@ -13,7 +13,6 @@ else ()
endif()
endif ()
-
#
# library setup
#
@@ -34,6 +33,25 @@ set (
MetaProgramming.h
)
+set (
+ UTILITIES_DEPENDS
+ CORSIKAgeometry
+ CORSIKAunits
+ C8::ext::boost # so far only for MetaProgramming
+ C8::ext::eigen3 # for COMboost
+ )
+
+if (TARGET cnpy)
+ LIST (APPEND
+ UTILITIES_HEADERS
+ SaveBoostHistogram.hpp
+ )
+ LIST (APPEND
+ UTILITIES_DEPENDS
+ cnpy # for SaveBoostHistogram
+ )
+endif (TARGET cnpy)
+
set (
UTILITIES_NAMESPACE
corsika/utl
@@ -53,10 +71,7 @@ set_target_properties (
# target dependencies on other libraries (also the header onlys)
target_link_libraries (
CORSIKAutilities
- CORSIKAgeometry
- CORSIKAunits
- C8::ext::boost # so far only for MetaProgramming
- C8::ext::eigen3 # for COMboost
+ ${UTILITIES_DEPENDS}
)
target_include_directories (
@@ -89,3 +104,12 @@ target_link_libraries (
CORSIKAutilities
CORSIKAtesting
)
+
+if (TARGET cnpy)
+ CORSIKA_ADD_TEST(testSaveBoostHistogram)
+ target_link_libraries (
+ testSaveBoostHistogram
+ CORSIKAutilities
+ CORSIKAtesting
+ )
+endif (TARGET cnpy)
diff --git a/Framework/Utilities/SaveBoostHistogram.hpp b/Framework/Utilities/SaveBoostHistogram.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..8418457f0ab8c9fe86412a77352cca278e919f35
--- /dev/null
+++ b/Framework/Utilities/SaveBoostHistogram.hpp
@@ -0,0 +1,104 @@
+#include <corsika/third_party/cnpy/cnpy.hpp>
+
+#include <boost/histogram.hpp>
+
+#include <functional>
+#include <memory>
+#include <numeric>
+#include <utility>
+#include <vector>
+
+namespace corsika {
+ namespace utl {
+
+ /**
+ * This functions saves a boost::histogram into a numpy file. Only rather basic axis
+ * types are supported: regular, variable, integer, category<int>. Only "ordinary" bin
+ * counts (i.e. a double or int) are supported, nothing fancy like profiles.
+ *
+ * Note that this function makes a temporary, dense copy of the histogram, which could
+ * be an issue for huge sizes (e.g. for high dimensions)
+ */
+ template <class Axes, class Storage>
+ void save_hist(boost::histogram::histogram<Axes, Storage> const& h,
+ std::string const& filename) {
+ int const rank = h.rank();
+
+ std::vector<size_t> axes_dims;
+ axes_dims.reserve(rank);
+
+ auto overflow = std::make_unique<bool[]>(rank);
+ auto underflow = std::make_unique<bool[]>(rank);
+
+ std::vector<char> axis_types;
+ axis_types.reserve(rank);
+
+ for (int i = 0; i < rank; ++i) {
+ auto const& ax = h.axis(i);
+ unsigned const has_underflow = (ax.options() & 0x01) ? 1 : 0;
+ unsigned const has_overflow = (ax.options() & 0x02) ? 1 : 0;
+
+ underflow[i] = has_underflow;
+ overflow[i] = has_overflow;
+
+ axes_dims.emplace_back(ax.size() + has_underflow + has_overflow);
+
+ if (ax.continuous()) {
+ axis_types.push_back('c');
+ std::vector<double> ax_edges;
+ ax_edges.reserve(ax.size() + 1);
+
+ for (int j = 0; j < ax.size(); ++j) { ax_edges.push_back(ax.bin(j).lower()); }
+
+ ax_edges.push_back(ax.bin(ax.size() - 1).upper());
+
+ cnpy::npz_save(filename, std::string{"binedges_"} + std::to_string(i),
+ ax_edges.data(), {ax_edges.size()}, "a");
+ } else {
+ axis_types.push_back('d');
+ std::vector<int64_t> bins; // we assume that discrete axes have integer bins
+ bins.reserve(ax.size());
+
+ for (int j = 0; j < ax.size(); ++j) { bins.push_back(ax.bin(j).lower()); }
+
+ cnpy::npz_save(filename, std::string{"bins_"} + std::to_string(i), bins.data(),
+ {bins.size()}, "a");
+ }
+
+ cnpy::npz_save(filename, std::string{"axistypes"}, axis_types.data(),
+ {axis_types.size()}, "a");
+
+ cnpy::npz_save(filename, std::string{"overflow"}, overflow.get(),
+ {axis_types.size()}, "a");
+ cnpy::npz_save(filename, std::string{"underflow"}, underflow.get(),
+ {axis_types.size()}, "a");
+ }
+
+ auto const prod_axis_size = std::accumulate(axes_dims.cbegin(), axes_dims.cend(),
+ unsigned{1}, std::multiplies<>());
+ auto temp = std::make_unique<float[]>(prod_axis_size);
+
+ assert(prod_axis_size == h.size());
+
+ // reduce multi-dim. to 1-dim, row-major (i.e., last axis index is contiguous in
+ // memory) take special care of underflow bins which have -1 as index and thus need
+ // to be shifted by +1
+
+ for (auto&& x : indexed(h, boost::histogram::coverage::all)) {
+ int p = 0;
+
+ for (int axis_index = 0; axis_index < rank; ++axis_index) {
+ int const offset_underflow = (h.axis(axis_index).options() & 0x01) ? 1 : 0;
+ auto k = x.index(axis_index) + offset_underflow;
+ p = k + p * axes_dims.at(axis_index);
+ }
+
+ temp[p] = *x;
+ }
+
+ cnpy::npz_save(filename, "data", temp.get(), axes_dims, "a");
+ // In Python this array can directly be assigned to a histogram view if that
+ // histogram has its axes correspondingly: hist.view(flow=True)[:] = file['data']
+ }
+ } // namespace utl
+} // namespace corsika
diff --git a/Framework/Utilities/testSaveBoostHistogram.cc b/Framework/Utilities/testSaveBoostHistogram.cc
new file mode 100644
index 0000000000000000000000000000000000000000..aa05a1947a9c989724dee5244f41859fa634f59f
--- /dev/null
+++ b/Framework/Utilities/testSaveBoostHistogram.cc
@@ -0,0 +1,39 @@
+/*
+ * (c) Copyright 2020 CORSIKA Project, corsika-project@lists.kit.edu
+ *
+ * This software is distributed under the terms of the GNU General Public
+ * Licence version 3 (GPL Version 3). See file LICENSE for a full version of
+ * the license.
+ */
+
+#include <catch2/catch.hpp>
+#include <corsika/utl/SaveBoostHistogram.hpp>
+
+#include <random>
+
+namespace bh = boost::histogram;
+
+TEST_CASE("SaveHistogram") {
+ std::mt19937 rng;
+ std::normal_distribution n1{2., 6.};
+ std::exponential_distribution e{1.};
+ std::uniform_int_distribution u{1, 10};
+ std::uniform_real_distribution<double> r{-3, 3};
+
+ auto h = bh::make_histogram(
+ bh::axis::regular{5, 0, 10, "normal"}, bh::axis::regular{3, 0, 4, "exponential"},
+ bh::axis::category<int>{{2, 3, 5, 7}, "integer category"},
+ bh::axis::regular<double, bh::use_default, bh::use_default,
+ bh::axis::option::growth_t>{10, -1, 1, "integer category"});
+
+ for (int i{0}; i < 100'000; ++i) {
+ auto const a = n1(rng);
+ auto const b = e(rng);
+ auto const c = u(rng);
+ auto const d = r(rng);
+
+ h(a, b, c, d);
+ }
+
+ corsika::utl::save_hist(h, "hist.npz");
+}
diff --git a/Processes/QGSJetII/QGSJetIIStack.h b/Processes/QGSJetII/QGSJetIIStack.h
index 8e3c87ddc9581464428809e86f8cce1dde87623a..04685ab7cf0a3efc89f6c953b6c3f0354758003c 100644
--- a/Processes/QGSJetII/QGSJetIIStack.h
+++ b/Processes/QGSJetII/QGSJetIIStack.h
@@ -86,7 +86,7 @@ namespace corsika::process::qgsjetII {
public:
void SetParticleData(const int vID, const corsika::units::si::HEPEnergyType vE,
const MomentumVector& vP,
- const corsika::units::si::HEPMassType vM) {
+ const corsika::units::si::HEPMassType) {
SetPID(vID);
SetEnergy(vE);
SetMomentum(vP);
@@ -95,7 +95,7 @@ namespace corsika::process::qgsjetII {
void SetParticleData(ParticleInterface<StackIteratorInterface>& /*parent*/,
const int vID, const corsika::units::si::HEPEnergyType vE,
const MomentumVector& vP,
- const corsika::units::si::HEPMassType vM) {
+ const corsika::units::si::HEPMassType) {
SetPID(vID);
SetEnergy(vE);
SetMomentum(vP);
diff --git a/Processes/QGSJetII/testQGSJetII.cc b/Processes/QGSJetII/testQGSJetII.cc
index d9a3dcda735ef3c2e941cc760b1f5389dc7bfd48..5acb717a51b77ef05d50b7eaf6b04e9bee674f2b 100644
--- a/Processes/QGSJetII/testQGSJetII.cc
+++ b/Processes/QGSJetII/testQGSJetII.cc
@@ -143,8 +143,17 @@ TEST_CASE("QgsjetIIInterface", "[processes]") {
[[maybe_unused]] const GrammageType length = model.GetInteractionLength(particle);
CHECK(length / (1_g / square(1_cm)) == Approx(93.47).margin(0.1));
- CHECK(view.GetSize() == 14);
- CHECK(sumCharge(view) == 2);
+
+ /***********************************
+ It as turned out already two times (#291 and #307) that the detailed output of
+ QGSJetII event generation depends on the gfortran version used. This is not reliable
+ and cannot be tested in a unit test here. One related problem was already found (#291)
+ and is realted to undefined behaviour in the evaluation of functions in logical
+ expressions. It is not clear if #307 is the same issue.
+
+ CHECK(view.GetSize() == 14);
+ CHECK(sumCharge(view) == 2);
+ ************************************/
auto const secMomSum = sumMomentum(view, projectileMomentum.GetCoordinateSystem());
CHECK((secMomSum - projectileMomentum).norm() / projectileMomentum.norm() ==
Approx(0).margin(1e-2));
diff --git a/ThirdParty/CMakeLists.txt b/ThirdParty/CMakeLists.txt
index 709539f6ea0a1acb3f677df8c7d2eedec42907bf..b30fdf95b5289100a86006d4bd70c652eb93ef4a 100644
--- a/ThirdParty/CMakeLists.txt
+++ b/ThirdParty/CMakeLists.txt
@@ -35,7 +35,7 @@ message (STATUS "USE_BOOST_C8='${USE_BOOST_C8}'")
add_library (C8::ext::boost INTERFACE IMPORTED GLOBAL)
if ("x_${USE_BOOST_C8}" STREQUAL "x_SYSTEM")
- find_package (Boost REQUIRED mp11 iterator core format interval optional type_index histogram)
+ find_package (Boost REQUIRED COMPONENTS mp11 iterator core format interval optional type_index histogram)
message (STATUS "Using system-level boost version ${Boost_VERSION} at ${Boost_INCLUDE_DIR}")
set_target_properties (
@@ -299,3 +299,14 @@ else (Boost_IOSTREAMS_FOUND)
$<BUILD_INTERFACE:${CONEX_INCLUDE_DIR}>
)
endif (Boost_IOSTREAMS_FOUND)
+
+
+# libz needed for cnpy, used for SaveHistograms
+find_package (ZLIB QUIET)
+
+if (ZLIB_FOUND)
+ message (STATUS "Found ZLIB. Build cnpy for SaveHistograms")
+ add_subdirectory (cnpy)
+else (ZLIB_FOUND)
+ message (WARNING "Did not find ZLIB. Cannot build cnpy for SaveHistograms")
+endif (ZLIB_FOUND)
diff --git a/ThirdParty/cnpy/CMakeLists.txt b/ThirdParty/cnpy/CMakeLists.txt
new file mode 100644
index 0000000000000000000000000000000000000000..4a115728d660361d0159c1c309efdabb332b51c3
--- /dev/null
+++ b/ThirdParty/cnpy/CMakeLists.txt
@@ -0,0 +1,45 @@
+set (
+ CNPY_SOURCES
+ cnpy.cpp
+ )
+
+set (
+ CNPY_HEADERS
+ cnpy.hpp
+ )
+
+set (
+ CNPY_NAMESPACE
+ corsika/third_party/cnpy
+ )
+
+add_library (cnpy STATIC ${CNPY_SOURCES})
+CORSIKA_COPY_HEADERS_TO_NAMESPACE (cnpy ${CNPY_NAMESPACE} ${CNPY_HEADERS})
+
+set_target_properties (
+ cnpy
+ PROPERTIES
+ VERSION ${PROJECT_VERSION}
+ SOVERSION 1
+ )
+
+# target dependencies on other libraries (also the header onlys)
+target_link_libraries (
+ cnpy
+ PUBLIC
+ ZLIB::ZLIB
+ )
+
+target_include_directories (
+ cnpy
+ INTERFACE
+ $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/include>
+ $<INSTALL_INTERFACE:include/include>
+ )
+
+install (
+ TARGETS cnpy
+ LIBRARY DESTINATION lib
+ ARCHIVE DESTINATION lib
+ )
+
diff --git a/ThirdParty/cnpy/cnpy.cpp b/ThirdParty/cnpy/cnpy.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..b4c26fccf411b18ae5b11d1c607b436dfdc75225
--- /dev/null
+++ b/ThirdParty/cnpy/cnpy.cpp
@@ -0,0 +1,340 @@
+//Copyright (C) 2011 Carl Rogers
+//Released under MIT License
+//license available in LICENSE file, or at http://www.opensource.org/licenses/mit-license.php
+
+#include "cnpy.hpp"
+#include <complex>
+#include <cstdlib>
+#include <algorithm>
+#include <cstring>
+#include <iomanip>
+#include <stdint.h>
+#include <stdexcept>
+#include <regex>
+
+char cnpy::BigEndianTest() {
+ int x = 1;
+ return (((char *)&x)[0]) ? '<' : '>';
+}
+
+char cnpy::map_type(const std::type_info& t)
+{
+ if(t == typeid(float) ) return 'f';
+ if(t == typeid(double) ) return 'f';
+ if(t == typeid(long double) ) return 'f';
+
+ if(t == typeid(int) ) return 'i';
+ if(t == typeid(char) ) return 'i';
+ if(t == typeid(short) ) return 'i';
+ if(t == typeid(long) ) return 'i';
+ if(t == typeid(long long) ) return 'i';
+
+ if(t == typeid(unsigned char) ) return 'u';
+ if(t == typeid(unsigned short) ) return 'u';
+ if(t == typeid(unsigned long) ) return 'u';
+ if(t == typeid(unsigned long long) ) return 'u';
+ if(t == typeid(unsigned int) ) return 'u';
+
+ if(t == typeid(bool) ) return 'b';
+
+ if(t == typeid(std::complex<float>) ) return 'c';
+ if(t == typeid(std::complex<double>) ) return 'c';
+ if(t == typeid(std::complex<long double>) ) return 'c';
+
+ else return '?';
+}
+
+template<> std::vector<char>& cnpy::operator+=(std::vector<char>& lhs, const std::string rhs) {
+ lhs.insert(lhs.end(),rhs.begin(),rhs.end());
+ return lhs;
+}
+
+template<> std::vector<char>& cnpy::operator+=(std::vector<char>& lhs, const char* rhs) {
+ //write in little endian
+ size_t len = strlen(rhs);
+ lhs.reserve(len);
+ for(size_t byte = 0; byte < len; byte++) {
+ lhs.push_back(rhs[byte]);
+ }
+ return lhs;
+}
+
+void cnpy::parse_npy_header(unsigned char* buffer,size_t& word_size, std::vector<size_t>& shape, bool& fortran_order) {
+ //std::string magic_string(buffer,6);
+ uint8_t major_version = *reinterpret_cast<uint8_t*>(buffer+6);
+ uint8_t minor_version = *reinterpret_cast<uint8_t*>(buffer+7);
+ uint16_t header_len = *reinterpret_cast<uint16_t*>(buffer+8);
+ std::string header(reinterpret_cast<char*>(buffer+9),header_len);
+
+ size_t loc1, loc2;
+
+ //fortran order
+ loc1 = header.find("fortran_order")+16;
+ fortran_order = (header.substr(loc1,4) == "True" ? true : false);
+
+ //shape
+ loc1 = header.find("(");
+ loc2 = header.find(")");
+
+ std::regex num_regex("[0-9][0-9]*");
+ std::smatch sm;
+ shape.clear();
+
+ std::string str_shape = header.substr(loc1+1,loc2-loc1-1);
+ while(std::regex_search(str_shape, sm, num_regex)) {
+ shape.push_back(std::stoi(sm[0].str()));
+ str_shape = sm.suffix().str();
+ }
+
+ //endian, word size, data type
+ //byte order code | stands for not applicable.
+ //not sure when this applies except for byte array
+ loc1 = header.find("descr")+9;
+ bool littleEndian = (header[loc1] == '<' || header[loc1] == '|' ? true : false);
+ assert(littleEndian);
+
+ //char type = header[loc1+1];
+ //assert(type == map_type(T));
+
+ std::string str_ws = header.substr(loc1+2);
+ loc2 = str_ws.find("'");
+ word_size = atoi(str_ws.substr(0,loc2).c_str());
+}
+
+void cnpy::parse_npy_header(FILE* fp, size_t& word_size, std::vector<size_t>& shape, bool& fortran_order) {
+ char buffer[256];
+ size_t res = fread(buffer,sizeof(char),11,fp);
+ if(res != 11)
+ throw std::runtime_error("parse_npy_header: failed fread");
+ std::string header = fgets(buffer,256,fp);
+ assert(header[header.size()-1] == '\n');
+
+ size_t loc1, loc2;
+
+ //fortran order
+ loc1 = header.find("fortran_order");
+ if (loc1 == std::string::npos)
+ throw std::runtime_error("parse_npy_header: failed to find header keyword: 'fortran_order'");
+ loc1 += 16;
+ fortran_order = (header.substr(loc1,4) == "True" ? true : false);
+
+ //shape
+ loc1 = header.find("(");
+ loc2 = header.find(")");
+ if (loc1 == std::string::npos || loc2 == std::string::npos)
+ throw std::runtime_error("parse_npy_header: failed to find header keyword: '(' or ')'");
+
+ std::regex num_regex("[0-9][0-9]*");
+ std::smatch sm;
+ shape.clear();
+
+ std::string str_shape = header.substr(loc1+1,loc2-loc1-1);
+ while(std::regex_search(str_shape, sm, num_regex)) {
+ shape.push_back(std::stoi(sm[0].str()));
+ str_shape = sm.suffix().str();
+ }
+
+ //endian, word size, data type
+ //byte order code | stands for not applicable.
+ //not sure when this applies except for byte array
+ loc1 = header.find("descr");
+ if (loc1 == std::string::npos)
+ throw std::runtime_error("parse_npy_header: failed to find header keyword: 'descr'");
+ loc1 += 9;
+ bool littleEndian = (header[loc1] == '<' || header[loc1] == '|' ? true : false);
+ assert(littleEndian);
+
+ //char type = header[loc1+1];
+ //assert(type == map_type(T));
+
+ std::string str_ws = header.substr(loc1+2);
+ loc2 = str_ws.find("'");
+ word_size = atoi(str_ws.substr(0,loc2).c_str());
+}
+
+void cnpy::parse_zip_footer(FILE* fp, uint16_t& nrecs, size_t& global_header_size, size_t& global_header_offset)
+{
+ std::vector<char> footer(22);
+ fseek(fp,-22,SEEK_END);
+ size_t res = fread(&footer[0],sizeof(char),22,fp);
+ if(res != 22)
+ throw std::runtime_error("parse_zip_footer: failed fread");
+
+ uint16_t disk_no, disk_start, nrecs_on_disk, comment_len;
+ disk_no = *(uint16_t*) &footer[4];
+ disk_start = *(uint16_t*) &footer[6];
+ nrecs_on_disk = *(uint16_t*) &footer[8];
+ nrecs = *(uint16_t*) &footer[10];
+ global_header_size = *(uint32_t*) &footer[12];
+ global_header_offset = *(uint32_t*) &footer[16];
+ comment_len = *(uint16_t*) &footer[20];
+
+ assert(disk_no == 0);
+ assert(disk_start == 0);
+ assert(nrecs_on_disk == nrecs);
+ assert(comment_len == 0);
+}
+
+cnpy::NpyArray load_the_npy_file(FILE* fp) {
+ std::vector<size_t> shape;
+ size_t word_size;
+ bool fortran_order;
+ cnpy::parse_npy_header(fp,word_size,shape,fortran_order);
+
+ cnpy::NpyArray arr(shape, word_size, fortran_order);
+ size_t nread = fread(arr.data<char>(),1,arr.num_bytes(),fp);
+ if(nread != arr.num_bytes())
+ throw std::runtime_error("load_the_npy_file: failed fread");
+ return arr;
+}
+
+cnpy::NpyArray load_the_npz_array(FILE* fp, uint32_t compr_bytes, uint32_t uncompr_bytes) {
+
+ std::vector<unsigned char> buffer_compr(compr_bytes);
+ std::vector<unsigned char> buffer_uncompr(uncompr_bytes);
+ size_t nread = fread(&buffer_compr[0],1,compr_bytes,fp);
+ if(nread != compr_bytes)
+ throw std::runtime_error("load_the_npy_file: failed fread");
+
+ int err;
+ z_stream d_stream;
+
+ d_stream.zalloc = Z_NULL;
+ d_stream.zfree = Z_NULL;
+ d_stream.opaque = Z_NULL;
+ d_stream.avail_in = 0;
+ d_stream.next_in = Z_NULL;
+ err = inflateInit2(&d_stream, -MAX_WBITS);
+
+ d_stream.avail_in = compr_bytes;
+ d_stream.next_in = &buffer_compr[0];
+ d_stream.avail_out = uncompr_bytes;
+ d_stream.next_out = &buffer_uncompr[0];
+
+ err = inflate(&d_stream, Z_FINISH);
+ err = inflateEnd(&d_stream);
+
+ std::vector<size_t> shape;
+ size_t word_size;
+ bool fortran_order;
+ cnpy::parse_npy_header(&buffer_uncompr[0],word_size,shape,fortran_order);
+
+ cnpy::NpyArray array(shape, word_size, fortran_order);
+
+ size_t offset = uncompr_bytes - array.num_bytes();
+ memcpy(array.data<unsigned char>(),&buffer_uncompr[0]+offset,array.num_bytes());
+
+ return array;
+}
+
+cnpy::npz_t cnpy::npz_load(std::string fname) {
+ FILE* fp = fopen(fname.c_str(),"rb");
+
+ if(!fp) {
+ throw std::runtime_error("npz_load: Error! Unable to open file "+fname+"!");
+ }
+
+ cnpy::npz_t arrays;
+
+ while(1) {
+ std::vector<char> local_header(30);
+ size_t headerres = fread(&local_header[0],sizeof(char),30,fp);
+ if(headerres != 30)
+ throw std::runtime_error("npz_load: failed fread");
+
+ //if we've reached the global header, stop reading
+ if(local_header[2] != 0x03 || local_header[3] != 0x04) break;
+
+ //read in the variable name
+ uint16_t name_len = *(uint16_t*) &local_header[26];
+ std::string varname(name_len,' ');
+ size_t vname_res = fread(&varname[0],sizeof(char),name_len,fp);
+ if(vname_res != name_len)
+ throw std::runtime_error("npz_load: failed fread");
+
+ //erase the lagging .npy
+ varname.erase(varname.end()-4,varname.end());
+
+ //read in the extra field
+ uint16_t extra_field_len = *(uint16_t*) &local_header[28];
+ if(extra_field_len > 0) {
+ std::vector<char> buff(extra_field_len);
+ size_t efield_res = fread(&buff[0],sizeof(char),extra_field_len,fp);
+ if(efield_res != extra_field_len)
+ throw std::runtime_error("npz_load: failed fread");
+ }
+
+ uint16_t compr_method = *reinterpret_cast<uint16_t*>(&local_header[0]+8);
+ uint32_t compr_bytes = *reinterpret_cast<uint32_t*>(&local_header[0]+18);
+ uint32_t uncompr_bytes = *reinterpret_cast<uint32_t*>(&local_header[0]+22);
+
+ if(compr_method == 0) {arrays[varname] = load_the_npy_file(fp);}
+ else {arrays[varname] = load_the_npz_array(fp,compr_bytes,uncompr_bytes);}
+ }
+
+ fclose(fp);
+ return arrays;
+}
+
+cnpy::NpyArray cnpy::npz_load(std::string fname, std::string varname) {
+ FILE* fp = fopen(fname.c_str(),"rb");
+
+ if(!fp) throw std::runtime_error("npz_load: Unable to open file "+fname);
+
+ while(1) {
+ std::vector<char> local_header(30);
+ size_t header_res = fread(&local_header[0],sizeof(char),30,fp);
+ if(header_res != 30)
+ throw std::runtime_error("npz_load: failed fread");
+
+ //if we've reached the global header, stop reading
+ if(local_header[2] != 0x03 || local_header[3] != 0x04) break;
+
+ //read in the variable name
+ uint16_t name_len = *(uint16_t*) &local_header[26];
+ std::string vname(name_len,' ');
+ size_t vname_res = fread(&vname[0],sizeof(char),name_len,fp);
+ if(vname_res != name_len)
+ throw std::runtime_error("npz_load: failed fread");
+ vname.erase(vname.end()-4,vname.end()); //erase the lagging .npy
+
+ //read in the extra field
+ uint16_t extra_field_len = *(uint16_t*) &local_header[28];
+ fseek(fp,extra_field_len,SEEK_CUR); //skip past the extra field
+
+ uint16_t compr_method = *reinterpret_cast<uint16_t*>(&local_header[0]+8);
+ uint32_t compr_bytes = *reinterpret_cast<uint32_t*>(&local_header[0]+18);
+ uint32_t uncompr_bytes = *reinterpret_cast<uint32_t*>(&local_header[0]+22);
+
+ if(vname == varname) {
+ NpyArray array = (compr_method == 0) ? load_the_npy_file(fp) : load_the_npz_array(fp,compr_bytes,uncompr_bytes);
+ fclose(fp);
+ return array;
+ }
+ else {
+ //skip past the data
+ uint32_t size = *(uint32_t*) &local_header[22];
+ fseek(fp,size,SEEK_CUR);
+ }
+ }
+
+ fclose(fp);
+
+ //if we get here, we haven't found the variable in the file
+ throw std::runtime_error("npz_load: Variable name "+varname+" not found in "+fname);
+}
+
+cnpy::NpyArray cnpy::npy_load(std::string fname) {
+
+ FILE* fp = fopen(fname.c_str(), "rb");
+
+ if(!fp) throw std::runtime_error("npy_load: Unable to open file "+fname);
+
+ NpyArray arr = load_the_npy_file(fp);
+
+ fclose(fp);
+ return arr;
+}
+
+
+
diff --git a/ThirdParty/cnpy/cnpy.hpp b/ThirdParty/cnpy/cnpy.hpp
new file mode 100644
index 0000000000000000000000000000000000000000..ffbcdb4b522b37b94f3d962bad9c9665d42b776b
--- /dev/null
+++ b/ThirdParty/cnpy/cnpy.hpp
@@ -0,0 +1,266 @@
+//Copyright (C) 2011 Carl Rogers
+//Released under MIT License
+//license available in LICENSE file, or at http://www.opensource.org/licenses/mit-license.php
+
+#pragma once
+
+#include<string>
+#include<stdexcept>
+#include<sstream>
+#include<vector>
+#include<cstdio>
+#include<typeinfo>
+#include<iostream>
+#include<cassert>
+#include<zlib.h>
+#include<map>
+#include<memory>
+#include<stdint.h>
+#include<numeric>
+
+namespace cnpy {
+
+ struct NpyArray {
+ NpyArray(const std::vector<size_t>& _shape, size_t _word_size, bool _fortran_order) :
+ shape(_shape), word_size(_word_size), fortran_order(_fortran_order)
+ {
+ num_vals = 1;
+ for(size_t i = 0;i < shape.size();i++) num_vals *= shape[i];
+ data_holder = std::shared_ptr<std::vector<char>>(
+ new std::vector<char>(num_vals * word_size));
+ }
+
+ NpyArray() : shape(0), word_size(0), fortran_order(0), num_vals(0) { }
+
+ template<typename T>
+ T* data() {
+ return reinterpret_cast<T*>(&(*data_holder)[0]);
+ }
+
+ template<typename T>
+ const T* data() const {
+ return reinterpret_cast<T*>(&(*data_holder)[0]);
+ }
+
+ template<typename T>
+ std::vector<T> as_vec() const {
+ const T* p = data<T>();
+ return std::vector<T>(p, p+num_vals);
+ }
+
+ size_t num_bytes() const {
+ return data_holder->size();
+ }
+
+ std::shared_ptr<std::vector<char>> data_holder;
+ std::vector<size_t> shape;
+ size_t word_size;
+ bool fortran_order;
+ size_t num_vals;
+ };
+
+ using npz_t = std::map<std::string, NpyArray>;
+
+ char BigEndianTest();
+ char map_type(const std::type_info& t);
+ template<typename T> std::vector<char> create_npy_header(const std::vector<size_t>& shape);
+ void parse_npy_header(FILE* fp,size_t& word_size, std::vector<size_t>& shape, bool& fortran_order);
+ void parse_npy_header(unsigned char* buffer,size_t& word_size, std::vector<size_t>& shape, bool& fortran_order);
+ void parse_zip_footer(FILE* fp, uint16_t& nrecs, size_t& global_header_size, size_t& global_header_offset);
+ npz_t npz_load(std::string fname);
+ NpyArray npz_load(std::string fname, std::string varname);
+ NpyArray npy_load(std::string fname);
+
+ template<typename T> std::vector<char>& operator+=(std::vector<char>& lhs, const T rhs) {
+ //write in little endian
+ for(size_t byte = 0; byte < sizeof(T); byte++) {
+ char val = *((char*)&rhs+byte);
+ lhs.push_back(val);
+ }
+ return lhs;
+ }
+
+ template<> std::vector<char>& operator+=(std::vector<char>& lhs, const std::string rhs);
+ template<> std::vector<char>& operator+=(std::vector<char>& lhs, const char* rhs);
+
+
+ template<typename T> void npy_save(std::string fname, const T* data, const std::vector<size_t> shape, std::string mode = "w") {
+ FILE* fp = NULL;
+ std::vector<size_t> true_data_shape; //if appending, the shape of existing + new data
+
+ if(mode == "a") fp = fopen(fname.c_str(),"r+b");
+
+ if(fp) {
+ //file exists. we need to append to it. read the header, modify the array size
+ size_t word_size;
+ bool fortran_order;
+ parse_npy_header(fp,word_size,true_data_shape,fortran_order);
+ assert(!fortran_order);
+
+ if(word_size != sizeof(T)) {
+ std::cout<<"libnpy error: "<<fname<<" has word size "<<word_size<<" but npy_save appending data sized "<<sizeof(T)<<"\n";
+ assert( word_size == sizeof(T) );
+ }
+ if(true_data_shape.size() != shape.size()) {
+ std::cout<<"libnpy error: npy_save attempting to append misdimensioned data to "<<fname<<"\n";
+ assert(true_data_shape.size() != shape.size());
+ }
+
+ for(size_t i = 1; i < shape.size(); i++) {
+ if(shape[i] != true_data_shape[i]) {
+ std::cout<<"libnpy error: npy_save attempting to append misshaped data to "<<fname<<"\n";
+ assert(shape[i] == true_data_shape[i]);
+ }
+ }
+ true_data_shape[0] += shape[0];
+ }
+ else {
+ fp = fopen(fname.c_str(),"wb");
+ true_data_shape = shape;
+ }
+
+ std::vector<char> header = create_npy_header<T>(true_data_shape);
+ size_t nels = std::accumulate(shape.begin(),shape.end(),1,std::multiplies<size_t>());
+
+ fseek(fp,0,SEEK_SET);
+ fwrite(&header[0],sizeof(char),header.size(),fp);
+ fseek(fp,0,SEEK_END);
+ fwrite(data,sizeof(T),nels,fp);
+ fclose(fp);
+ }
+
+ template<typename T> void npz_save(std::string zipname, std::string fname, const T* data, const std::vector<size_t>& shape, std::string mode = "w")
+ {
+ //first, append a .npy to the fname
+ fname += ".npy";
+
+ //now, on with the show
+ FILE* fp = NULL;
+ uint16_t nrecs = 0;
+ size_t global_header_offset = 0;
+ std::vector<char> global_header;
+
+ if(mode == "a") fp = fopen(zipname.c_str(),"r+b");
+
+ if(fp) {
+ //zip file exists. we need to add a new npy file to it.
+ //first read the footer. this gives us the offset and size of the global header
+ //then read and store the global header.
+ //below, we will write the the new data at the start of the global header then append the global header and footer below it
+ size_t global_header_size;
+ parse_zip_footer(fp,nrecs,global_header_size,global_header_offset);
+ fseek(fp,global_header_offset,SEEK_SET);
+ global_header.resize(global_header_size);
+ size_t res = fread(&global_header[0],sizeof(char),global_header_size,fp);
+ if(res != global_header_size){
+ throw std::runtime_error("npz_save: header read error while adding to existing zip");
+ }
+ fseek(fp,global_header_offset,SEEK_SET);
+ }
+ else {
+ fp = fopen(zipname.c_str(),"wb");
+ }
+
+ std::vector<char> npy_header = create_npy_header<T>(shape);
+
+ size_t nels = std::accumulate(shape.begin(),shape.end(),1,std::multiplies<size_t>());
+ size_t nbytes = nels*sizeof(T) + npy_header.size();
+
+ //get the CRC of the data to be added
+ uint32_t crc = crc32(0L,(uint8_t*)&npy_header[0],npy_header.size());
+ crc = crc32(crc,(uint8_t*)data,nels*sizeof(T));
+
+ //build the local header
+ std::vector<char> local_header;
+ local_header += "PK"; //first part of sig
+ local_header += (uint16_t) 0x0403; //second part of sig
+ local_header += (uint16_t) 20; //min version to extract
+ local_header += (uint16_t) 0; //general purpose bit flag
+ local_header += (uint16_t) 0; //compression method
+ local_header += (uint16_t) 0; //file last mod time
+ local_header += (uint16_t) 0; //file last mod date
+ local_header += (uint32_t) crc; //crc
+ local_header += (uint32_t) nbytes; //compressed size
+ local_header += (uint32_t) nbytes; //uncompressed size
+ local_header += (uint16_t) fname.size(); //fname length
+ local_header += (uint16_t) 0; //extra field length
+ local_header += fname;
+
+ //build global header
+ global_header += "PK"; //first part of sig
+ global_header += (uint16_t) 0x0201; //second part of sig
+ global_header += (uint16_t) 20; //version made by
+ global_header.insert(global_header.end(),local_header.begin()+4,local_header.begin()+30);
+ global_header += (uint16_t) 0; //file comment length
+ global_header += (uint16_t) 0; //disk number where file starts
+ global_header += (uint16_t) 0; //internal file attributes
+ global_header += (uint32_t) 0; //external file attributes
+ global_header += (uint32_t) global_header_offset; //relative offset of local file header, since it begins where the global header used to begin
+ global_header += fname;
+
+ //build footer
+ std::vector<char> footer;
+ footer += "PK"; //first part of sig
+ footer += (uint16_t) 0x0605; //second part of sig
+ footer += (uint16_t) 0; //number of this disk
+ footer += (uint16_t) 0; //disk where footer starts
+ footer += (uint16_t) (nrecs+1); //number of records on this disk
+ footer += (uint16_t) (nrecs+1); //total number of records
+ footer += (uint32_t) global_header.size(); //nbytes of global headers
+ footer += (uint32_t) (global_header_offset + nbytes + local_header.size()); //offset of start of global headers, since global header now starts after newly written array
+ footer += (uint16_t) 0; //zip file comment length
+
+ //write everything
+ fwrite(&local_header[0],sizeof(char),local_header.size(),fp);
+ fwrite(&npy_header[0],sizeof(char),npy_header.size(),fp);
+ fwrite(data,sizeof(T),nels,fp);
+ fwrite(&global_header[0],sizeof(char),global_header.size(),fp);
+ fwrite(&footer[0],sizeof(char),footer.size(),fp);
+ fclose(fp);
+ }
+
+ template<typename T> void npy_save(std::string fname, const std::vector<T> data, std::string mode = "w") {
+ std::vector<size_t> shape;
+ shape.push_back(data.size());
+ npy_save(fname, &data[0], shape, mode);
+ }
+
+ template<typename T> void npz_save(std::string zipname, std::string fname, const std::vector<T> data, std::string mode = "w") {
+ std::vector<size_t> shape;
+ shape.push_back(data.size());
+ npz_save(zipname, fname, &data[0], shape, mode);
+ }
+
+ template<typename T> std::vector<char> create_npy_header(const std::vector<size_t>& shape) {
+
+ std::vector<char> dict;
+ dict += "{'descr': '";
+ dict += BigEndianTest();
+ dict += map_type(typeid(T));
+ dict += std::to_string(sizeof(T));
+ dict += "', 'fortran_order': False, 'shape': (";
+ dict += std::to_string(shape[0]);
+ for(size_t i = 1;i < shape.size();i++) {
+ dict += ", ";
+ dict += std::to_string(shape[i]);
+ }
+ if(shape.size() == 1) dict += ",";
+ dict += "), }";
+ //pad with spaces so that preamble+dict is modulo 16 bytes. preamble is 10 bytes. dict needs to end with \n
+ int remainder = 16 - (10 + dict.size()) % 16;
+ dict.insert(dict.end(),remainder,' ');
+ dict.back() = '\n';
+
+ std::vector<char> header;
+ header += (char) 0x93;
+ header += "NUMPY";
+ header += (char) 0x01; //major version of numpy format
+ header += (char) 0x00; //minor version of numpy format
+ header += (uint16_t) dict.size();
+ header.insert(header.end(),dict.begin(),dict.end());
+
+ return header;
+ }
+
+
+}
diff --git a/Tools/CMakeLists.txt b/Tools/CMakeLists.txt
index c1347108d1d451651bb8a6a593c4acd7c8aab66d..109b6d44abe7794a1f9ee39456ae24d883872e30 100644
--- a/Tools/CMakeLists.txt
+++ b/Tools/CMakeLists.txt
@@ -1,4 +1,4 @@
-set (TOOLS_FILES plot_tracks.sh plot_crossings.sh)
+set (TOOLS_FILES plot_tracks.sh plot_crossings.sh read_hist.py)
install (
FILES ${TOOLS_FILES}
diff --git a/Tools/read_hist.py b/Tools/read_hist.py
new file mode 100755
index 0000000000000000000000000000000000000000..4348e118bdcb4fe4bc3e11fcd50b626425238788
--- /dev/null
+++ b/Tools/read_hist.py
@@ -0,0 +1,29 @@
+import numpy as np
+import matplotlib.pyplot as plt
+import boost_histogram as bh
+import operator
+import functools
+
+def read_hist(filename):
+ """
+ read numpy file produced with CORSIKA 8's save_hist() function into
+ boost-histogram object.
+ """
+
+ d = np.load(filename)
+ axistypes = d['axistypes'].view('c')
+ overflow = d['overflow']
+ underflow = d['underflow']
+
+ axes = []
+ for i, (at, has_overflow, has_underflow) in enumerate(zip(axistypes, overflow, underflow)):
+ if at == b'c':
+ axes.append(bh.axis.Variable(d[f'binedges_{i}'], overflow=has_overflow, underflow=has_underflow))
+ elif at == b'd':
+ axes.append(bh.axis.IntCategory(d[f'bins_{i}'], growth=(not has_overflow)))
+
+ h = bh.Histogram(*axes)
+ h.view(flow=True)[:] = d['data']
+
+ return h
+