IAP GITLAB

Skip to content
Snippets Groups Projects
Commit f3cc1227 authored by Remy Prechelt's avatar Remy Prechelt
Browse files

Enable ZSTD compression for output files.

parent 51f948f5
No related branches found
No related tags found
No related merge requests found
...@@ -6,6 +6,7 @@ boost/1.74.0 ...@@ -6,6 +6,7 @@ boost/1.74.0
zlib/1.2.11 zlib/1.2.11
yaml-cpp/0.6.3 yaml-cpp/0.6.3
arrow/2.0.0 arrow/2.0.0
zstd/1.4.9
[generators] [generators]
cmake cmake
...@@ -13,7 +14,7 @@ cmake ...@@ -13,7 +14,7 @@ cmake
[options] [options]
arrow:shared=False arrow:shared=False
arrow:parquet=True arrow:parquet=True
arrow:with_zlib=True arrow:with_zlib=False
arrow:fPIC=False arrow:fPIC=False
arrow:with_re2=False arrow:with_re2=False
arrow:with_protobuf=False arrow:with_protobuf=False
......
...@@ -19,6 +19,9 @@ namespace corsika { ...@@ -19,6 +19,9 @@ namespace corsika {
// setup the streamer // setup the streamer
output_.initStreamer((directory / "particles.parquet").string()); output_.initStreamer((directory / "particles.parquet").string());
// enable compression with the default level
output_.enableCompression();
// build the schema // build the schema
output_.addField("pdg", parquet::Repetition::REQUIRED, parquet::Type::INT32, output_.addField("pdg", parquet::Repetition::REQUIRED, parquet::Type::INT32,
parquet::ConvertedType::INT_32); parquet::ConvertedType::INT_32);
......
...@@ -30,6 +30,11 @@ namespace corsika { ...@@ -30,6 +30,11 @@ namespace corsika {
fields_.push_back(parquet::schema::PrimitiveNode::Make(args...)); fields_.push_back(parquet::schema::PrimitiveNode::Make(args...));
} }
void ParquetStreamer::enableCompression(int const level) {
builder_.compression(parquet::Compression::ZSTD);
builder_.compression_level(level);
}
void ParquetStreamer::buildStreamer() { void ParquetStreamer::buildStreamer() {
// build the top level schema // build the top level schema
...@@ -47,8 +52,6 @@ namespace corsika { ...@@ -47,8 +52,6 @@ namespace corsika {
outfile_->Close(); outfile_->Close();
} }
std::shared_ptr<parquet::StreamWriter> ParquetStreamer::getWriter() { std::shared_ptr<parquet::StreamWriter> ParquetStreamer::getWriter() { return writer_; }
return writer_;
}
} // namespace corsika } // namespace corsika
...@@ -50,6 +50,11 @@ namespace corsika { ...@@ -50,6 +50,11 @@ namespace corsika {
template <typename... TArgs> template <typename... TArgs>
void addField(TArgs&&... args); void addField(TArgs&&... args);
/**
* Enable compression for this streamer.
*/
void enableCompression(int const level = 3);
/** /**
* Finalize the streamer construction. * Finalize the streamer construction.
*/ */
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment