diff -Nru c-blosc-1.17.1+ds1/ANNOUNCE.rst c-blosc-1.21.1+ds2/ANNOUNCE.rst --- c-blosc-1.17.1+ds1/ANNOUNCE.rst 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/ANNOUNCE.rst 2021-10-06 08:16:29.000000000 +0000 @@ -1,13 +1,15 @@ =============================================================== - Announcing C-Blosc 1.17.1 + Announcing C-Blosc 1.21.1 A blocking, shuffling and lossless compression library for C =============================================================== What is new? ============ -A maintenance release where LZ4 and Zstd internal codecs have been updated -to latest versions. +This is a maintenance release. Fix pthread flag when linking on ppc64le. +Vendored BloscLZ, Zlib and Zstd codecs have been updated to their latest +versions too; this can bring important performance improvements, so if +speed is a priority to you, an upgrade is recommended. For more info, please see the release notes in: diff -Nru c-blosc-1.17.1+ds1/bench/bench.c c-blosc-1.21.1+ds2/bench/bench.c --- c-blosc-1.17.1+ds1/bench/bench.c 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/bench/bench.c 2021-10-06 08:16:29.000000000 +0000 @@ -171,7 +171,7 @@ void do_bench(char *compressor, char *shuffle, int nthreads, int size, int elsize, - int rshift, int unsafe, FILE * ofile) { + int rshift, FILE * ofile) { void *src, *srccpy; void *dest[NCHUNKS], *dest2; int nbytes = 0, cbytes = 0; @@ -215,13 +215,13 @@ if (retcode) abort(); } - fprintf(ofile, "--> %d, %d, %d, %d, %s, %s, %s\n", nthreads, size, elsize, - rshift, compressor, shuffle, unsafe ? "unsafe" : "safe"); + fprintf(ofile, "--> %d, %d, %d, %d, %s, %s\n", nthreads, size, elsize, + rshift, compressor, shuffle); fprintf(ofile, "********************** Run info ******************************\n"); fprintf(ofile, "Blosc version: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE); fprintf(ofile, "Using synthetic data with %d significant bits (out of 32)\n", rshift); fprintf(ofile, "Dataset size: %d bytes\tType size: %d bytes\n", size, elsize); - fprintf(ofile, "Working set: %.1f MB\t\t", (size*nchunks) / (float)MB); + fprintf(ofile, "Working set: %.1f MB\t\t", (size * nchunks) / (float)MB); fprintf(ofile, "Number of threads: %d\n", nthreads); fprintf(ofile, "********************** Running benchmarks *********************\n"); @@ -283,8 +283,7 @@ nbytes = size; } else { - nbytes = unsafe ? blosc_decompress_unsafe(dest[j], dest2, size) - : blosc_decompress(dest[j], dest2, size); + nbytes = blosc_decompress(dest[j], dest2, size); } } } @@ -378,12 +377,11 @@ int extreme_suite = 0; int debug_suite = 0; int nthreads = 4; /* The number of threads */ - int size = 2*MB; /* Buffer size */ + int size = 4 * MB; /* Buffer size */ int elsize = 8; /* Datatype size */ int rshift = 19; /* Significant bits */ - int workingset = 256*MB; /* The maximum allocated memory */ + int workingset = 256 * MB; /* The maximum allocated memory */ int nthreads_, size_, elsize_, rshift_, i; - int unsafe = 1; FILE * output_file = stdout; blosc_timestamp_t last, current; float totaltime; @@ -394,7 +392,7 @@ strncpy(usage, "Usage: bench [blosclz | lz4 | lz4hc | snappy | zlib | zstd] " "[noshuffle | shuffle | bitshuffle] " "[single | suite | hardsuite | extremesuite | debugsuite] " - "[nthreads] [bufsize(bytes)] [typesize] [sbits] [unsafe | safe]", 255); + "[nthreads] [bufsize(bytes)] [typesize] [sbits]", 255); if (argc < 2) { printf("%s\n", usage); @@ -492,18 +490,7 @@ rshift = atoi(argv[7]); } - if (argc >= 9) { - if (strcmp(argv[8], "unsafe") == 0) { - unsafe = 1; - } else if (strcmp(argv[8], "safe") == 0) { - unsafe = 0; - } else { - printf("%s\n", usage); - exit(1); - } - } - - if ((argc >= 10) || !(single || suite || hard_suite || extreme_suite)) { + if ((argc >= 9) || !(single || suite || hard_suite || extreme_suite)) { printf("%s\n", usage); exit(1); } @@ -515,7 +502,7 @@ if (suite) { for (nthreads_=1; nthreads_ <= nthreads; nthreads_++) { - do_bench(compressor, shuffle, nthreads_, size, elsize, rshift, unsafe, output_file); + do_bench(compressor, shuffle, nthreads_, size, elsize, rshift, output_file); } } else if (hard_suite) { @@ -530,7 +517,7 @@ nchunks = get_nchunks(size_+i, workingset); niter = 1; for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) { - do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, unsafe, output_file); + do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, output_file); blosc_set_timestamp(¤t); totaltime = (float)getseconds(last, current); printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n", @@ -549,7 +536,7 @@ for (size_ = 32*KB; size_ <= size; size_ *= 2) { nchunks = get_nchunks(size_+i, workingset); for (nthreads_ = 1; nthreads_ <= nthreads; nthreads_++) { - do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, unsafe, output_file); + do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, output_file); blosc_set_timestamp(¤t); totaltime = (float)getseconds(last, current); printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n", @@ -568,7 +555,7 @@ for (size_ = size; size_ <= 16*MB; size_ *= 2) { nchunks = get_nchunks(size_+i, workingset); for (nthreads_ = nthreads; nthreads_ <= 6; nthreads_++) { - do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, unsafe, output_file); + do_bench(compressor, shuffle, nthreads_, size_+i, elsize_, rshift_, output_file); blosc_set_timestamp(¤t); totaltime = (float)getseconds(last, current); printf("Elapsed time:\t %6.1f s. Processed data: %.1f GB\n", @@ -581,7 +568,7 @@ } /* Single mode */ else { - do_bench(compressor, shuffle, nthreads, size, elsize, rshift, unsafe, output_file); + do_bench(compressor, shuffle, nthreads, size, elsize, rshift, output_file); } /* Print out some statistics */ diff -Nru c-blosc-1.17.1+ds1/bench/plot-speeds.py c-blosc-1.21.1+ds2/bench/plot-speeds.py --- c-blosc-1.17.1+ds1/bench/plot-speeds.py 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/bench/plot-speeds.py 2021-10-06 08:16:29.000000000 +0000 @@ -29,16 +29,12 @@ tmp = line.split('-->')[1] parts = tmp.split(', ') nthreads, size, elsize, sbits, codec, shuffle = parts[:6] - safe = 'unsafe' - if len(parts) > 6: - safe = parts[6] nthreads, size, elsize, sbits = map(int, (nthreads, size, elsize, sbits)) - values["size"] = size * NCHUNKS / MB_ + values["size"] = size / MB_ values["elsize"] = elsize values["sbits"] = sbits values["codec"] = codec values["shuffle"] = shuffle - values["safe"] = safe # New run for nthreads (ratios, speedsw, speedsr) = ([], [], []) # Add a new entry for (ratios, speedw, speedr) @@ -47,21 +43,21 @@ elif line.startswith('memcpy(write):'): tmp = line.split(',')[1] memcpyw = float(tmp.split(' ')[1]) - values["memcpyw"].append(memcpyw) + values["memcpyw"].append(memcpyw / 1024) elif line.startswith('memcpy(read):'): tmp = line.split(',')[1] memcpyr = float(tmp.split(' ')[1]) - values["memcpyr"].append(memcpyr) + values["memcpyr"].append(memcpyr / 1024) elif line.startswith('comp(write):'): tmp = line.split(',')[1] speedw = float(tmp.split(' ')[1]) ratio = float(line.split(':')[-1]) - speedsw.append(speedw) + speedsw.append(speedw / 1024) ratios.append(ratio) elif line.startswith('decomp(read):'): tmp = line.split(',')[1] speedr = float(tmp.split(' ')[1]) - speedsr.append(speedr) + speedsr.append(speedr / 1024) if "OK" not in line: print("WARNING! OK not found in decomp line!") @@ -71,7 +67,7 @@ def show_plot(plots, yaxis, legends, gtitle, xmax=None, ymax=None): xlabel('Compresssion ratio') - ylabel('Speed (MB/s)') + ylabel('Speed (GB/s)') title(gtitle) xlim(0, xmax) ylim(0, ymax) @@ -190,7 +186,7 @@ if options.title: plot_title = options.title else: - plot_title += " (%(size).1f MB, %(elsize)d bytes, %(sbits)d bits), %(codec)s %(shuffle)s %(safe)s" % values + plot_title += " (%(size).1f MB, %(elsize)d bytes, %(sbits)d bits), %(codec)s %(shuffle)s" % values gtitle = plot_title @@ -219,7 +215,7 @@ mean = np.mean(values["memcpyr"]) message = "memcpy (read from memory)" plot_ = axhline(mean, linewidth=3, linestyle='-.', color='black') - text(4.0, mean+400, message) + text(4.0, mean+.4, message) plots.append(plot_) show_plot(plots, yaxis, legends, gtitle, xmax=int(options.xmax) if options.xmax else None, diff -Nru c-blosc-1.17.1+ds1/blosc/blosc.c c-blosc-1.21.1+ds2/blosc/blosc.c --- c-blosc-1.17.1+ds1/blosc/blosc.c 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/blosc/blosc.c 2021-10-06 08:16:29.000000000 +0000 @@ -423,10 +423,10 @@ char* output, size_t maxout, int clevel) { int cbytes; - if (input_length > (size_t)(2<<30)) - return -1; /* input larger than 1 GB is not supported */ + if (input_length > (size_t)(UINT32_C(2)<<30)) + return -1; /* input larger than 2 GB is not supported */ /* clevel for lz4hc goes up to 12, at least in LZ4 1.7.5 - * but levels larger than 9 does not buy much compression. */ + * but levels larger than 9 do not buy much compression. */ cbytes = LZ4_compress_HC(input, output, (int)input_length, (int)maxout, clevel); return cbytes; @@ -522,8 +522,7 @@ } #endif /* HAVE_ZSTD */ -static int initialize_decompress_func(struct blosc_context* context, - int unsafe) { +static int initialize_decompress_func(struct blosc_context* context) { int8_t header_flags = *(context->header_flags); int32_t compformat = (header_flags & 0xe0) >> 5; int compversion = context->compversion; @@ -532,8 +531,7 @@ if (compversion != BLOSC_BLOSCLZ_VERSION_FORMAT) { return -9; } - context->decompress_func = - unsafe ? &blosclz_decompress_unsafe : &blosclz_decompress; + context->decompress_func = &blosclz_decompress; return 0; } #if defined(HAVE_LZ4) @@ -652,9 +650,8 @@ } } if (context->compcode == BLOSC_BLOSCLZ) { - int doshuffle = (*(context->header_flags) & BLOSC_DOSHUFFLE) && (typesize > 1); cbytes = blosclz_compress(context->clevel, _tmp+j*neblock, neblock, - dest, maxout, doshuffle); + dest, maxout, !dont_split); } #if defined(HAVE_LZ4) else if (context->compcode == BLOSC_LZ4) { @@ -712,7 +709,7 @@ if ((ntbytes+neblock) > maxbytes) { return 0; /* Non-compressible data */ } - blosc_internal_fastcopy(dest, _tmp + j * neblock, neblock); + fastcopy(dest, _tmp + j * neblock, neblock); cbytes = neblock; } _sw32(dest - 4, cbytes); @@ -776,7 +773,7 @@ src = base_src + src_offset; /* Uncompress */ if (cbytes == neblock) { - blosc_internal_fastcopy(_tmp, src, neblock); + fastcopy(_tmp, src, neblock); nbytes = neblock; } else { @@ -830,8 +827,8 @@ if (context->compress) { if (*(context->header_flags) & BLOSC_MEMCPYED) { /* We want to memcpy only */ - blosc_internal_fastcopy(context->dest + BLOSC_MAX_OVERHEAD + j * context->blocksize, - context->src + j * context->blocksize, bsize); + fastcopy(context->dest + BLOSC_MAX_OVERHEAD + j * context->blocksize, + context->src + j * context->blocksize, bsize); cbytes = bsize; } else { @@ -848,8 +845,8 @@ else { if (*(context->header_flags) & BLOSC_MEMCPYED) { /* We want to memcpy only */ - blosc_internal_fastcopy(context->dest + j * context->blocksize, - context->src + BLOSC_MAX_OVERHEAD + j * context->blocksize, bsize); + fastcopy(context->dest + j * context->blocksize, + context->src + BLOSC_MAX_OVERHEAD + j * context->blocksize, bsize); cbytes = bsize; } else { @@ -880,7 +877,9 @@ (void)rc; // just to avoid 'unused-variable' warning /* Check whether we need to restart threads */ - blosc_set_nthreads_(context); + if (blosc_set_nthreads_(context) < 0) { + return -1; + } /* Set sentinels */ context->thread_giveup_code = 1; @@ -1034,15 +1033,20 @@ /* Enlarge the blocksize for splittable codecs */ if (clevel > 0 && split_block(context->compcode, typesize, blocksize)) { - if (blocksize > (1 << 16)) { - /* Do not use a too large split buffer (> 64 KB) for splitting codecs */ - blocksize = (1 << 16); + if (blocksize > (1 << 18)) { + /* Do not use a too large split buffer (> 256 KB) for splitting codecs */ + blocksize = (1 << 18); } blocksize *= typesize; if (blocksize < (1 << 16)) { /* Do not use a too small blocksize (< 64 KB) when typesize is small */ blocksize = (1 << 16); } + if (blocksize > 1024 * 1024) { + /* But do not exceed 1 MB per thread (having this capacity in L3 is normal in modern CPUs) */ + blocksize = 1024 * 1024; + } + } /* Check that blocksize is not too large */ @@ -1070,6 +1074,8 @@ int32_t blocksize, int32_t numthreads) { + char *envvar = NULL; + int warnlvl = 0; /* Set parameters */ context->compress = 1; context->src = (const uint8_t*)src; @@ -1083,21 +1089,29 @@ context->end_threads = 0; context->clevel = clevel; + envvar = getenv("BLOSC_WARN"); + if (envvar != NULL) { + warnlvl = strtol(envvar, NULL, 10); + } + /* Check buffer size limits */ if (sourcesize > BLOSC_MAX_BUFFERSIZE) { - fprintf(stderr, "Input buffer size cannot exceed %d bytes\n", - BLOSC_MAX_BUFFERSIZE); - return -1; + if (warnlvl > 0) { + fprintf(stderr, "Input buffer size cannot exceed %d bytes\n", + BLOSC_MAX_BUFFERSIZE); + } + return 0; } if (destsize < BLOSC_MAX_OVERHEAD) { - fprintf(stderr, "Output buffer size should be larger than %d bytes\n", - BLOSC_MAX_OVERHEAD); - return -1; + if (warnlvl > 0) { + fprintf(stderr, "Output buffer size should be larger than %d bytes\n", + BLOSC_MAX_OVERHEAD); + } + return 0; } /* Compression level */ if (clevel < 0 || clevel > 9) { - /* If clevel not in 0..9, print an error */ fprintf(stderr, "`clevel` parameter must be between 0 and 9!\n"); return -10; } @@ -1273,10 +1287,10 @@ nbytes, src, dest, destsize, blosc_compname_to_compcode(compressor), blocksize, numinternalthreads); - if (error < 0) { return error; } + if (error <= 0) { return error; } error = write_compression_header(&context, clevel, doshuffle); - if (error < 0) { return error; } + if (error <= 0) { return error; } result = blosc_compress_context(&context); @@ -1395,10 +1409,10 @@ typesize, nbytes, src, dest, destsize, g_compressor, g_force_blocksize, g_threads); - if (result < 0) { break; } + if (result <= 0) { break; } result = write_compression_header(g_global_context, clevel, doshuffle); - if (result < 0) { break; } + if (result <= 0) { break; } result = blosc_compress_context(g_global_context); } while (0); @@ -1412,8 +1426,7 @@ const void* src, void* dest, size_t destsize, - int numinternalthreads, - int unsafe) + int numinternalthreads) { uint8_t version; int32_t ntbytes; @@ -1475,7 +1488,7 @@ return -1; } } else { - ntbytes = initialize_decompress_func(context, unsafe); + ntbytes = initialize_decompress_func(context); if (ntbytes != 0) return ntbytes; /* Validate that compressed size is large enough to hold the bstarts array */ @@ -1494,16 +1507,14 @@ return ntbytes; } -/* Implementation of blosc_decompress_ctx{,_unsafe}. */ -static int blosc_decompress_ctx_impl(const void* src, void* dest, - size_t destsize, int numinternalthreads, - int unsafe) { +int blosc_decompress_ctx(const void* src, void* dest, size_t destsize, + int numinternalthreads) { int result; struct blosc_context context; context.threads_started = 0; result = blosc_run_decompression_with_context(&context, src, dest, destsize, - numinternalthreads, unsafe); + numinternalthreads); if (numinternalthreads > 1) { @@ -1513,21 +1524,7 @@ return result; } -int blosc_decompress_ctx(const void* src, void* dest, size_t destsize, - int numinternalthreads) { - return blosc_decompress_ctx_impl(src, dest, destsize, numinternalthreads, - /*unsafe=*/0); -} - -int blosc_decompress_ctx_unsafe(const void* src, void* dest, size_t destsize, - int numinternalthreads) { - return blosc_decompress_ctx_impl(src, dest, destsize, numinternalthreads, - /*unsafe=*/1); -} - -/* Implementation of blosc_decompress{,_unsafe}. */ -static int blosc_decompress_impl(const void* src, void* dest, - size_t destsize, int unsafe) { +int blosc_decompress(const void* src, void* dest, size_t destsize) { int result; char* envvar; long nthreads; @@ -1557,26 +1554,14 @@ pthread_mutex_lock(global_comp_mutex); result = blosc_run_decompression_with_context(g_global_context, src, dest, - destsize, g_threads, unsafe); + destsize, g_threads); pthread_mutex_unlock(global_comp_mutex); return result; } -/* The public routine for decompression. See blosc.h for docstrings. */ -int blosc_decompress(const void *src, void *dest, size_t destsize) { - return blosc_decompress_impl(src, dest, destsize, /*unsafe=*/0); -} - -int blosc_decompress_unsafe(const void *src, void *dest, size_t destsize) { - return blosc_decompress_impl(src, dest, destsize, /*unsafe=*/1); -} - - -/* Implementation of blosc_getitem{,_unsafe}. */ -static int blosc_getitem_impl(const void* src, int start, int nitems, - void* dest, int unsafe) { +int blosc_getitem(const void* src, int start, int nitems, void* dest) { uint8_t *_src=NULL; /* current pos for source buffer */ uint8_t version, compversion; /* versions for compressed header */ uint8_t flags; /* flags for header */ @@ -1629,7 +1614,7 @@ return -1; } } else { - ntbytes = initialize_decompress_func(&context, /*unsafe=*/unsafe); + ntbytes = initialize_decompress_func(&context); if (ntbytes != 0) return ntbytes; if (nblocks >= (compressedsize - 16) / 4) { @@ -1682,8 +1667,8 @@ /* Do the actual data copy */ if (flags & BLOSC_MEMCPYED) { /* We want to memcpy only */ - blosc_internal_fastcopy((uint8_t *) dest + ntbytes, - (uint8_t *) src + BLOSC_MAX_OVERHEAD + j * blocksize + startb, bsize2); + fastcopy((uint8_t *) dest + ntbytes, + (uint8_t *) src + BLOSC_MAX_OVERHEAD + j * blocksize + startb, bsize2); cbytes = bsize2; } else { @@ -1696,7 +1681,7 @@ break; } /* Copy to destination */ - blosc_internal_fastcopy((uint8_t *) dest + ntbytes, tmp2 + startb, bsize2); + fastcopy((uint8_t *) dest + ntbytes, tmp2 + startb, bsize2); cbytes = bsize2; } ntbytes += cbytes; @@ -1707,17 +1692,6 @@ return ntbytes; } -/* Specific routine optimized for decompression a small number of - items out of a compressed chunk. This does not use threads because - it would affect negatively to performance. */ -int blosc_getitem(const void *src, int start, int nitems, void *dest) { - return blosc_getitem_impl(src, start, nitems, dest, /*unsafe=*/0); -} - -int blosc_getitem_unsafe(const void *src, int start, int nitems, void *dest) { - return blosc_getitem_impl(src, start, nitems, dest, /*unsafe=*/1); -} - /* Decompress & unshuffle several blocks in a single thread */ static void *t_blosc(void *ctxt) { @@ -1817,8 +1791,8 @@ if (compress) { if (flags & BLOSC_MEMCPYED) { /* We want to memcpy only */ - blosc_internal_fastcopy(dest + BLOSC_MAX_OVERHEAD + nblock_ * blocksize, src + nblock_ * blocksize, - bsize); + fastcopy(dest + BLOSC_MAX_OVERHEAD + nblock_ * blocksize, + src + nblock_ * blocksize, bsize); cbytes = bsize; } else { @@ -1830,8 +1804,8 @@ else { if (flags & BLOSC_MEMCPYED) { /* We want to memcpy only */ - blosc_internal_fastcopy(dest + nblock_ * blocksize, src + BLOSC_MAX_OVERHEAD + nblock_ * blocksize, - bsize); + fastcopy(dest + nblock_ * blocksize, + src + BLOSC_MAX_OVERHEAD + nblock_ * blocksize, bsize); cbytes = bsize; } else { @@ -1873,7 +1847,7 @@ /* End of critical section */ /* Copy the compressed buffer to destination */ - blosc_internal_fastcopy(dest + ntdest, tmp2, cbytes); + fastcopy(dest + ntdest, tmp2, cbytes); } else { nblock_++; @@ -2004,7 +1978,9 @@ /* Launch a new pool of threads */ if (context->numthreads > 1 && context->numthreads != context->threads_started) { blosc_release_threadpool(context); - init_threads(context); + if (init_threads(context) < 0) { + return -1; + } } /* We have now started the threads */ diff -Nru c-blosc-1.17.1+ds1/blosc/blosc-common.h c-blosc-1.21.1+ds2/blosc/blosc-common.h --- c-blosc-1.17.1+ds1/blosc/blosc-common.h 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/blosc/blosc-common.h 2021-10-06 08:16:29.000000000 +0000 @@ -12,9 +12,12 @@ #include "blosc-export.h" #include +#ifdef __GNUC__ +#define BLOSC_GCC_VERSION (__GNUC__ * 100 + __GNUC_MINOR__) +#endif // __GNUC__ + /* Import standard integer type definitions */ #if defined(_WIN32) && !defined(__MINGW32__) - /* stdint.h only available in VS2010 (VC++ 16.0) and newer */ #if defined(_MSC_VER) && _MSC_VER < 1600 #include "win32/stdint-windows.h" diff -Nru c-blosc-1.17.1+ds1/blosc/blosc.h c-blosc-1.21.1+ds2/blosc/blosc.h --- c-blosc-1.17.1+ds1/blosc/blosc.h 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/blosc/blosc.h 2021-10-06 08:16:29.000000000 +0000 @@ -18,14 +18,14 @@ /* Version numbers */ #define BLOSC_VERSION_MAJOR 1 /* for major interface/format changes */ -#define BLOSC_VERSION_MINOR 17 /* for minor interface/format changes */ +#define BLOSC_VERSION_MINOR 21 /* for minor interface/format changes */ #define BLOSC_VERSION_RELEASE 1 /* for tweaks, bug-fixes, or development */ -#define BLOSC_VERSION_STRING "1.17.1" /* string version. Sync with above! */ +#define BLOSC_VERSION_STRING "1.21.1" /* string version. Sync with above! */ #define BLOSC_VERSION_REVISION "$Rev$" /* revision version */ -#define BLOSC_VERSION_DATE "$Date:: 2019-12-12 #$" /* date version */ +#define BLOSC_VERSION_DATE "$Date:: 2021-10-06 #$" /* date version */ -#define BLOSCLZ_VERSION_STRING "2.0.0" /* the internal compressor version */ +#define BLOSCLZ_VERSION_STRING "2.5.1" /* the internal compressor version */ /* The *_FORMAT symbols should be just 1-byte long */ #define BLOSC_VERSION_FORMAT 2 /* Blosc format version, starting at 1 */ @@ -215,6 +215,10 @@ This will call blosc_set_splitmode() with the different supported values. See blosc_set_splitmode() docstrings for more info on each mode. + BLOSC_WARN=(INTEGER): This will print some warning message on stderr + showing more info in situations where data inputs cannot be compressed. + The values can range from 1 (less verbose) to 10 (full verbose). 0 is + the same as if the BLOSC_WARN envvar was not defined. */ BLOSC_EXPORT int blosc_compress(int clevel, int doshuffle, size_t typesize, size_t nbytes, const void *src, void *dest, @@ -278,17 +282,6 @@ BLOSC_EXPORT int blosc_decompress(const void *src, void *dest, size_t destsize); /** - Same as `blosc_decompress`, except that this is not safe to run on - untrusted/possibly corrupted input (even after calling - `blosc_cbuffer_validate`). - - This may be marginally faster than `blosc_decompress` due to skipping certain - bounds checking and validation. -*/ -BLOSC_EXPORT int blosc_decompress_unsafe(const void* src, void* dest, - size_t destsize); - -/** Context interface to blosc decompression. This does not require a call to blosc_init() and can be called from multithreaded applications without the global lock being used, so allowing Blosc @@ -311,18 +304,6 @@ size_t destsize, int numinternalthreads); /** - Same as `blosc_decompress_ctx`, except that this is not safe to run on - untrusted/possibly corrupted input (even after calling - `blosc_cbuffer_validate`). - - This may be marginally faster than `blosc_decompress_ctx` due to skipping - certain bounds checking and validation. -*/ -BLOSC_EXPORT int blosc_decompress_ctx_unsafe(const void* src, void* dest, - size_t destsize, - int numinternalthreads); - -/** Get `nitems` (of typesize size) in `src` buffer starting in `start`. The items are returned in `dest` buffer, which has to have enough space for storing all items. @@ -333,17 +314,6 @@ BLOSC_EXPORT int blosc_getitem(const void *src, int start, int nitems, void *dest); /** - Same as `blosc_getitem`, except that this is not safe to run on - untrusted/possibly corrupted input (even after calling - `blosc_cbuffer_validate`). - - This may be marginally faster than `blosc_getitem` due to skipping certain - bounds checking and validation. -*/ -BLOSC_EXPORT int blosc_getitem_unsafe(const void* src, int start, int nitems, - void* dest); - -/** Returns the current number of threads that are used for compression/decompression. */ @@ -369,7 +339,7 @@ /** Select the compressor to be used. The supported ones are "blosclz", - "lz4", "lz4hc", "snappy", "zlib" and "ztsd". If this function is not + "lz4", "lz4hc", "snappy", "zlib" and "zstd". If this function is not called, then "blosclz" will be used by default. In case the compressor is not recognized, or there is not support diff -Nru c-blosc-1.17.1+ds1/blosc/blosclz.c c-blosc-1.21.1+ds2/blosc/blosclz.c --- c-blosc-1.17.1+ds1/blosc/blosclz.c 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/blosc/blosclz.c 2021-10-06 08:16:29.000000000 +0000 @@ -1,10 +1,11 @@ /********************************************************************* Blosc - Blocked Shuffling and Compression Library - Author: Francesc Alted - Creation date: 2009-05-20 + Copyright (C) 2021 The Blosc Developers + https://blosc.org + License: BSD 3-Clause (see LICENSE.txt) - See LICENSES/BLOSC.txt for details about copyright and rights to use. + See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ /********************************************************************* @@ -14,35 +15,21 @@ #include -#include - -#if defined(_WIN32) && !defined(__MINGW32__) - #include - /* stdint.h only available in VS2010 (VC++ 16.0) and newer */ - #if defined(_MSC_VER) && _MSC_VER < 1600 - #include "win32/stdint-windows.h" - #else - #include - #endif -#else - #include -#endif /* _WIN32 */ - +#include #include "blosclz.h" #include "fastcopy.h" #include "blosc-common.h" -#include "blosc-comp-features.h" /* * Give hints to the compiler for branch prediction optimization. */ #if defined(__GNUC__) && (__GNUC__ > 2) -#define BLOSCLZ_EXPECT_CONDITIONAL(c) (__builtin_expect((c), 1)) -#define BLOSCLZ_UNEXPECT_CONDITIONAL(c) (__builtin_expect((c), 0)) +#define BLOSCLZ_LIKELY(c) (__builtin_expect((c), 1)) +#define BLOSCLZ_UNLIKELY(c) (__builtin_expect((c), 0)) #else -#define BLOSCLZ_EXPECT_CONDITIONAL(c) (c) -#define BLOSCLZ_UNEXPECT_CONDITIONAL(c) (c) +#define BLOSCLZ_LIKELY(c) (c) +#define BLOSCLZ_UNLIKELY(c) (c) #endif /* @@ -65,90 +52,44 @@ #endif #define HASH_LOG (14U) - -/* Simple, but pretty effective hash function for 3-byte sequence */ -// This is the original hash function used in fastlz -//#define HASH_FUNCTION(v, p, h) { \ -// v = BLOSCLZ_READU16(p); \ -// v ^= BLOSCLZ_READU16(p + 1) ^ ( v >> (16 - h)); \ -// v &= (1 << h) - 1; \ -//} +#define HASH_LOG2 (12U) // This is used in LZ4 and seems to work pretty well here too -#define HASH_FUNCTION(v, p, h) \ - v = ((BLOSCLZ_READU32(p) * 2654435761U) >> (32U - h)) - - -#define LITERAL(ip, op, op_limit, anchor, copy) { \ - if (BLOSCLZ_UNEXPECT_CONDITIONAL(op + 2 > op_limit)) \ - goto out; \ - *op++ = *anchor++; \ - ip = anchor; \ - copy++; \ - if (BLOSCLZ_UNEXPECT_CONDITIONAL(copy == MAX_COPY)) { \ - copy = 0; \ - *op++ = MAX_COPY-1; \ - nmax_copies++; \ - if (nmax_copies > max_nmax_copies) \ - goto out; \ - } \ - continue; \ +#define HASH_FUNCTION(v, s, h) { \ + v = (s * 2654435761U) >> (32U - h); \ } -#define IP_BOUNDARY 2 +#if defined(__AVX2__) +static uint8_t *get_run_32(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) { + uint8_t x = ip[-1]; -static uint8_t *get_run(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) { - uint8_t x = ip[-1]; - int64_t value, value2; - /* Broadcast the value for every byte in a 64-bit register */ - memset(&value, x, 8); - /* safe because the outer check against ip limit */ - while (ip < (ip_bound - sizeof(int64_t))) { -#if defined(BLOSC_STRICT_ALIGN) - memcpy(&value2, ref, 8); -#else - value2 = ((int64_t*)ref)[0]; -#endif - if (value != value2) { - /* Return the byte that starts to differ */ - while (*ref++ == x) ip++; - return ip; - } - else { - ip += 8; - ref += 8; - } - } - /* Look into the remainder */ - while ((ip < ip_bound) && (*ref++ == x)) ip++; - return ip; + while (ip < (ip_bound - (sizeof(__m256i)))) { + __m256i value, value2, cmp; + /* Broadcast the value for every byte in a 256-bit register */ + memset(&value, x, sizeof(__m256i)); + value2 = _mm256_loadu_si256((__m256i *)ref); + cmp = _mm256_cmpeq_epi64(value, value2); + if ((unsigned)_mm256_movemask_epi8(cmp) != 0xFFFFFFFF) { + /* Return the byte that starts to differ */ + while (*ref++ == x) ip++; + return ip; + } + else { + ip += sizeof(__m256i); + ref += sizeof(__m256i); + } + } + /* Look into the remainder */ + while ((ip < ip_bound) && (*ref++ == x)) ip++; + return ip; } +#endif -#ifdef __SSE2__ +#if defined(__SSE2__) static uint8_t *get_run_16(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) { uint8_t x = ip[-1]; - if (ip < (ip_bound - sizeof(int64_t))) { - int64_t value, value2; - /* Broadcast the value for every byte in a 64-bit register */ - memset(&value, x, 8); -#if defined(BLOSC_STRICT_ALIGN) - memcpy(&value2, ref, 8); -#else - value2 = ((int64_t*)ref)[0]; -#endif - if (value != value2) { - /* Return the byte that starts to differ */ - while (*ref++ == x) ip++; - return ip; - } - else { - ip += 8; - ref += 8; - } - } - /* safe because the outer check against ip limit */ while (ip < (ip_bound - sizeof(__m128i))) { __m128i value, value2, cmp; /* Broadcast the value for every byte in a 128-bit register */ @@ -169,17 +110,17 @@ while ((ip < ip_bound) && (*ref++ == x)) ip++; return ip; } + #endif -#ifdef __AVX2__ -static uint8_t *get_run_32(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) { +static uint8_t *get_run(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) { uint8_t x = ip[-1]; + int64_t value, value2; + /* Broadcast the value for every byte in a 64-bit register */ + memset(&value, x, 8); /* safe because the outer check against ip limit */ - if (ip < (ip_bound - sizeof(int64_t))) { - int64_t value, value2; - /* Broadcast the value for every byte in a 64-bit register */ - memset(&value, x, 8); + while (ip < (ip_bound - sizeof(int64_t))) { #if defined(BLOSC_STRICT_ALIGN) memcpy(&value2, ref, 8); #else @@ -195,42 +136,10 @@ ref += 8; } } - if (ip < (ip_bound - sizeof(__m128i))) { - __m128i value, value2, cmp; - /* Broadcast the value for every byte in a 128-bit register */ - memset(&value, x, sizeof(__m128i)); - value2 = _mm_loadu_si128((__m128i *) ref); - cmp = _mm_cmpeq_epi32(value, value2); - if (_mm_movemask_epi8(cmp) != 0xFFFF) { - /* Return the byte that starts to differ */ - while (*ref++ == x) ip++; - return ip; - } else { - ip += sizeof(__m128i); - ref += sizeof(__m128i); - } - } - while (ip < (ip_bound - (sizeof(__m256i)))) { - __m256i value, value2, cmp; - /* Broadcast the value for every byte in a 256-bit register */ - memset(&value, x, sizeof(__m256i)); - value2 = _mm256_loadu_si256((__m256i *)ref); - cmp = _mm256_cmpeq_epi64(value, value2); - if (_mm256_movemask_epi8(cmp) != 0xFFFFFFFF) { - /* Return the byte that starts to differ */ - while (*ref++ == x) ip++; - return ip; - } - else { - ip += sizeof(__m256i); - ref += sizeof(__m256i); - } - } /* Look into the remainder */ while ((ip < ip_bound) && (*ref++ == x)) ip++; return ip; } -#endif /* Return the byte that starts to differ */ @@ -258,23 +167,14 @@ static uint8_t *get_match_16(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) { __m128i value, value2, cmp; - if (ip < (ip_bound - sizeof(int64_t))) { - if (*(int64_t *) ref != *(int64_t *) ip) { - /* Return the byte that starts to differ */ - while (*ref++ == *ip++) {} - return ip; - } else { - ip += sizeof(int64_t); - ref += sizeof(int64_t); - } - } while (ip < (ip_bound - sizeof(__m128i))) { value = _mm_loadu_si128((__m128i *) ip); value2 = _mm_loadu_si128((__m128i *) ref); cmp = _mm_cmpeq_epi32(value, value2); if (_mm_movemask_epi8(cmp) != 0xFFFF) { /* Return the byte that starts to differ */ - return get_match(ip, ip_bound, ref); + while (*ref++ == *ip++) {} + return ip; } else { ip += sizeof(__m128i); @@ -291,36 +191,12 @@ #if defined(__AVX2__) static uint8_t *get_match_32(uint8_t *ip, const uint8_t *ip_bound, const uint8_t *ref) { - if (ip < (ip_bound - sizeof(int64_t))) { - if (*(int64_t *) ref != *(int64_t *) ip) { - /* Return the byte that starts to differ */ - while (*ref++ == *ip++) {} - return ip; - } else { - ip += sizeof(int64_t); - ref += sizeof(int64_t); - } - } - if (ip < (ip_bound - sizeof(__m128i))) { - __m128i value, value2, cmp; - value = _mm_loadu_si128((__m128i *) ip); - value2 = _mm_loadu_si128((__m128i *) ref); - cmp = _mm_cmpeq_epi32(value, value2); - if (_mm_movemask_epi8(cmp) != 0xFFFF) { - /* Return the byte that starts to differ */ - return get_match_16(ip, ip_bound, ref); - } - else { - ip += sizeof(__m128i); - ref += sizeof(__m128i); - } - } while (ip < (ip_bound - sizeof(__m256i))) { __m256i value, value2, cmp; value = _mm256_loadu_si256((__m256i *) ip); value2 = _mm256_loadu_si256((__m256i *)ref); cmp = _mm256_cmpeq_epi64(value, value2); - if (_mm256_movemask_epi8(cmp) != 0xFFFFFFFF) { + if ((unsigned)_mm256_movemask_epi8(cmp) != 0xFFFFFFFF) { /* Return the byte that starts to differ */ while (*ref++ == *ip++) {} return ip; @@ -337,118 +213,324 @@ #endif -int blosclz_compress(const int opt_level, const void* input, int length, - void* output, int maxout, int shuffle) { - uint8_t* ip = (uint8_t*)input; - uint8_t* ibase = (uint8_t*)input; - uint8_t* ip_bound = ip + length - IP_BOUNDARY; - uint8_t* ip_limit = ip + length - 12; - uint8_t* op = (uint8_t*)output; - uint8_t* op_limit; - uint16_t htab[1U << (uint8_t)HASH_LOG]; - int32_t hval; - uint8_t copy; - uint32_t nmax_copies = 0; - unsigned i; - uint8_t hashlog_[10] = {0, HASH_LOG - 4, HASH_LOG - 4, HASH_LOG - 3 , HASH_LOG - 2, - HASH_LOG - 1, HASH_LOG, HASH_LOG, HASH_LOG, HASH_LOG}; - uint8_t hashlog = hashlog_[opt_level]; - // The maximum amount of consecutive MAX_COPY copies before giving up - // 0 means something very close to RLE - uint8_t max_nmax_copies_[10] = {255U, 0U, 8U, 8U, 16U, 32U, 32U, 32U, 32U, 64U}; // 255 never used - uint8_t max_nmax_copies = max_nmax_copies_[opt_level]; - double maxlength_[10] = {-1, .1, .2, .3, .4, .6, .9, .95, 1.0, 1.0}; - int32_t maxlength = (int32_t)(length * maxlength_[opt_level]); - - if (maxlength > (int32_t)maxout) { - maxlength = (int32_t)maxout; +static uint8_t* get_run_or_match(uint8_t* ip, const uint8_t* ip_bound, const uint8_t* ref, bool run) { + if (BLOSCLZ_UNLIKELY(run)) { +#if defined(__AVX2__) + // Extensive experiments on AMD Ryzen3 say that regular get_run is faster + // ip = get_run_32(ip, ip_bound, ref); + ip = get_run(ip, ip_bound, ref); +#elif defined(__SSE2__) + // Extensive experiments on AMD Ryzen3 say that regular get_run is faster + // ip = get_run_16(ip, ip_bound, ref); + ip = get_run(ip, ip_bound, ref); +#else + ip = get_run(ip, ip_bound, ref); +#endif } - op_limit = op + maxlength; + else { +#if defined(__AVX2__) + // Extensive experiments on AMD Ryzen3 say that regular get_match_16 is faster + // ip = get_match_32(ip, ip_bound, ref); + ip = get_match_16(ip, ip_bound, ref); +#elif defined(__SSE2__) + ip = get_match_16(ip, ip_bound, ref); +#else + ip = get_match(ip, ip_bound, ref); +#endif + } + + return ip; +} + + +#define LITERAL(ip, op, op_limit, anchor, copy) { \ + if (BLOSCLZ_UNLIKELY(op + 2 > op_limit)) \ + goto out; \ + *op++ = *anchor++; \ + ip = anchor; \ + copy++; \ + if (BLOSCLZ_UNLIKELY(copy == MAX_COPY)) { \ + copy = 0; \ + *op++ = MAX_COPY-1; \ + } \ +} + +#define LITERAL2(ip, anchor, copy) { \ + oc++; anchor++; \ + ip = anchor; \ + copy++; \ + if (BLOSCLZ_UNLIKELY(copy == MAX_COPY)) { \ + copy = 0; \ + oc++; \ + } \ +} + +#define MATCH_SHORT(op, op_limit, len, distance) { \ + if (BLOSCLZ_UNLIKELY(op + 2 > op_limit)) \ + goto out; \ + *op++ = (uint8_t)((len << 5U) + (distance >> 8U)); \ + *op++ = (uint8_t)((distance & 255U)); \ +} + +#define MATCH_LONG(op, op_limit, len, distance) { \ + if (BLOSCLZ_UNLIKELY(op + 1 > op_limit)) \ + goto out; \ + *op++ = (uint8_t)((7U << 5U) + (distance >> 8U)); \ + for (len -= 7; len >= 255; len -= 255) { \ + if (BLOSCLZ_UNLIKELY(op + 1 > op_limit)) \ + goto out; \ + *op++ = 255; \ + } \ + if (BLOSCLZ_UNLIKELY(op + 2 > op_limit)) \ + goto out; \ + *op++ = (uint8_t)len; \ + *op++ = (uint8_t)((distance & 255U)); \ +} + +#define MATCH_SHORT_FAR(op, op_limit, len, distance) { \ + if (BLOSCLZ_UNLIKELY(op + 4 > op_limit)) \ + goto out; \ + *op++ = (uint8_t)((len << 5U) + 31); \ + *op++ = 255; \ + *op++ = (uint8_t)(distance >> 8U); \ + *op++ = (uint8_t)(distance & 255U); \ +} + +#define MATCH_LONG_FAR(op, op_limit, len, distance) { \ + if (BLOSCLZ_UNLIKELY(op + 1 > op_limit)) \ + goto out; \ + *op++ = (7U << 5U) + 31; \ + for (len -= 7; len >= 255; len -= 255) { \ + if (BLOSCLZ_UNLIKELY(op + 1 > op_limit)) \ + goto out; \ + *op++ = 255; \ + } \ + if (BLOSCLZ_UNLIKELY(op + 4 > op_limit)) \ + goto out; \ + *op++ = (uint8_t)len; \ + *op++ = 255; \ + *op++ = (uint8_t)(distance >> 8U); \ + *op++ = (uint8_t)(distance & 255U); \ +} + + +// Get a guess for the compressed size of a buffer +static double get_cratio(uint8_t* ibase, int maxlen, int minlen, int ipshift) { + uint8_t* ip = ibase; + int32_t oc = 0; + const uint16_t hashlen = (1U << (uint8_t)HASH_LOG2); + uint16_t htab[1U << (uint8_t)HASH_LOG2]; + uint32_t hval; + uint32_t seq; + uint8_t copy; + // Make a tradeoff between testing too much and too little + uint16_t limit = (maxlen > hashlen) ? hashlen : maxlen; + uint8_t* ip_bound = ibase + limit - 1; + uint8_t* ip_limit = ibase + limit - 12; // Initialize the hash table to distances of 0 - for (i = 0; i < (1U << hashlog); i++) { - htab[i] = 0; + memset(htab, 0, hashlen * sizeof(uint16_t)); + + /* we start with literal copy */ + copy = 4; + oc += 5; + + /* main loop */ + while (BLOSCLZ_LIKELY(ip < ip_limit)) { + const uint8_t* ref; + unsigned distance; + uint8_t* anchor = ip; /* comparison starting-point */ + + /* find potential match */ + seq = BLOSCLZ_READU32(ip); + HASH_FUNCTION(hval, seq, HASH_LOG2) + ref = ibase + htab[hval]; + + /* calculate distance to the match */ + distance = (unsigned int)(anchor - ref); + + /* update hash table */ + htab[hval] = (uint16_t) (anchor - ibase); + + if (distance == 0 || (distance >= MAX_FARDISTANCE)) { + LITERAL2(ip, anchor, copy) + continue; + } + + /* is this a match? check the first 4 bytes */ + if (BLOSCLZ_READU32(ref) == BLOSCLZ_READU32(ip)) { + ref += 4; + } + else { + /* no luck, copy as a literal */ + LITERAL2(ip, anchor, copy) + continue; + } + + /* last matched byte */ + ip = anchor + 4; + + /* distance is biased */ + distance--; + + /* get runs or matches; zero distance means a run */ + ip = get_run_or_match(ip, ip_bound, ref, !distance); + + ip -= ipshift; + unsigned len = (int)(ip - anchor); + if (len < minlen) { + LITERAL2(ip, anchor, copy) + continue; + } + + /* if we haven't copied anything, adjust the output counter */ + if (!copy) + oc--; + /* reset literal counter */ + copy = 0; + + /* encode the match */ + if (distance < MAX_DISTANCE) { + if (len >= 7) { + oc += ((len - 7) / 255) + 1; + } + oc += 2; + } + else { + /* far away, but not yet in the another galaxy... */ + if (len >= 7) { + oc += ((len - 7) / 255) + 1; + } + oc += 4; + } + + /* update the hash at match boundary */ + seq = BLOSCLZ_READU32(ip); + HASH_FUNCTION(hval, seq, HASH_LOG2) + htab[hval] = (uint16_t)(ip++ - ibase); + ip++; + /* assuming literal copy */ + oc++; } - /* output buffer cannot be less than 66 bytes or we can get into trouble */ - if (BLOSCLZ_UNEXPECT_CONDITIONAL(maxout < 66 || length < 4)) { + double ic; + ic = (double)(ip - ibase); + return ic / (double)oc; +} + + +int blosclz_compress(const int clevel, const void* input, int length, + void* output, int maxout, const int split_block) { + uint8_t* ibase = (uint8_t*)input; + + // Experiments say that checking 1/4 of the buffer is enough to figure out approx cratio + int maxlen = length / 4; + // Start probing somewhere inside the buffer + int shift = length - maxlen; + // Actual entropy probing! + double cratio = get_cratio(ibase + shift, maxlen, 3, 3); + // discard probes with small compression ratios (too expensive) + double cratio_[10] = {0, 2, 1.5, 1.2, 1.2, 1.2, 1.2, 1.15, 1.1, 1.0}; + if (cratio < cratio_[clevel]) { + goto out; + } + + /* When we go back in a match (shift), we obtain quite different compression properties. + * It looks like 4 is more useful in combination with bitshuffle and small typesizes + * Fallback to 4 because it provides more consistent results for large cratios. + * + * In this block we also check cratios for the beginning of the buffers and + * eventually discard those that are small (take too long to decompress). + * This process is called _entropy probing_. + */ + unsigned ipshift = 4; + // Compute optimal shift and minimum lengths for encoding + // Use 4 by default, except for low entropy data, where we should do a best effort + unsigned minlen = 4; + // BloscLZ works better with splits mostly, so when data is not split, do a best effort + // Why using cratio < 4 is based in experiments with low and high entropy + if (!split_block || cratio < 4) { + ipshift = 3; + minlen = 3; + } + + uint8_t hashlog_[10] = {0, HASH_LOG - 2, HASH_LOG - 1, HASH_LOG, HASH_LOG, + HASH_LOG, HASH_LOG, HASH_LOG, HASH_LOG, HASH_LOG}; + uint8_t hashlog = hashlog_[clevel]; + + uint8_t* ip = ibase; + const uint8_t* ip_bound = ibase + length - 1; + const uint8_t* ip_limit = ibase + length - 12; + uint8_t* op = (uint8_t*)output; + const uint8_t* op_limit = op + maxout; + + /* input and output buffer cannot be less than 16 and 66 bytes or we can get into trouble */ + if (length < 16 || maxout < 66) { return 0; } + // Initialize the hash table + uint32_t htab[1U << (uint8_t)HASH_LOG]; + memset(htab, 0, (1U << hashlog) * sizeof(uint32_t)); + /* we start with literal copy */ - copy = 2; + uint8_t copy = 4; *op++ = MAX_COPY - 1; *op++ = *ip++; *op++ = *ip++; + *op++ = *ip++; + *op++ = *ip++; /* main loop */ - while (BLOSCLZ_EXPECT_CONDITIONAL(ip < ip_limit)) { + while (BLOSCLZ_LIKELY(ip < ip_limit)) { const uint8_t* ref; - uint32_t distance; - uint32_t len = 3; /* minimum match length */ + unsigned distance; uint8_t* anchor = ip; /* comparison starting-point */ - /* check for a run */ - if (ip[0] == ip[-1] && BLOSCLZ_READU16(ip - 1) == BLOSCLZ_READU16(ip + 1)) { - distance = 1; - ref = anchor - 1 + 3; - goto match; - } - /* find potential match */ - HASH_FUNCTION(hval, ip, hashlog); + uint32_t seq = BLOSCLZ_READU32(ip); + uint32_t hval; + HASH_FUNCTION(hval, seq, hashlog) ref = ibase + htab[hval]; /* calculate distance to the match */ - distance = (int32_t)(anchor - ref); + distance = (unsigned int)(anchor - ref); - /* update hash table if necessary */ - /* not exactly sure why masking the distance works best, but this is what the experiments say */ - if (!shuffle || (distance & (MAX_COPY - 1)) == 0) { - htab[hval] = (uint16_t) (anchor - ibase); - } + /* update hash table */ + htab[hval] = (uint32_t) (anchor - ibase); if (distance == 0 || (distance >= MAX_FARDISTANCE)) { LITERAL(ip, op, op_limit, anchor, copy) + continue; } /* is this a match? check the first 4 bytes */ - if (BLOSCLZ_READU32(ref) == BLOSCLZ_READU32(ip)) { - len = 4; + if (BLOSCLZ_UNLIKELY(BLOSCLZ_READU32(ref) == BLOSCLZ_READU32(ip))) { ref += 4; - } - /* check just the first 3 bytes */ - else if (*ref++ != *ip++ || *ref++ != *ip++ || *ref++ != *ip) { + } else { /* no luck, copy as a literal */ LITERAL(ip, op, op_limit, anchor, copy) + continue; } - match: - /* last matched byte */ - ip = anchor + len; + ip = anchor + 4; /* distance is biased */ distance--; - if (!distance) { - /* zero distance means a run */ -#if defined(__AVX2__) - ip = get_run_32(ip, ip_bound, ref); -#elif defined(__SSE2__) - ip = get_run_16(ip, ip_bound, ref); -#else - ip = get_run(ip, ip_bound, ref); -#endif - } - else { -#if defined(__AVX2__) - ip = get_match_32(ip, ip_bound + IP_BOUNDARY, ref); -#elif defined(__SSE2__) - ip = get_match_16(ip, ip_bound + IP_BOUNDARY, ref); -#else - ip = get_match(ip, ip_bound + IP_BOUNDARY, ref); -#endif + /* get runs or matches; zero distance means a run */ + ip = get_run_or_match(ip, ip_bound, ref, !distance); + + /* length is biased, '1' means a match of 3 bytes */ + ip -= ipshift; + + unsigned len = (int)(ip - anchor); + + // Encoding short lengths is expensive during decompression + if (len < minlen || (len <= 5 && distance >= MAX_DISTANCE)) { + LITERAL(ip, op, op_limit, anchor, copy) + continue; } /* if we have copied something, adjust the copy count */ @@ -458,72 +540,54 @@ else /* back, to overwrite the copy count */ op--; - /* reset literal counter */ copy = 0; - /* length is biased, '1' means a match of 3 bytes */ - ip -= 3; - len = (int32_t)(ip - anchor); - - /* check that we have space enough to encode the match for all the cases */ - if (BLOSCLZ_UNEXPECT_CONDITIONAL(op + (len / 255) + 6 > op_limit)) goto out; - /* encode the match */ if (distance < MAX_DISTANCE) { - if (len < 7U) { - *op++ = (uint8_t)((len << 5U) + (distance >> 8U)); - *op++ = (uint8_t)((distance & 255U)); - } - else { - *op++ = (uint8_t)((7U << 5U) + (distance >> 8U)); - for (len -= 7U; len >= 255U; len -= 255U) - *op++ = 255U; - *op++ = (uint8_t)len; - *op++ = (uint8_t)((distance & 255U)); + if (len < 7) { + MATCH_SHORT(op, op_limit, len, distance) + } else { + MATCH_LONG(op, op_limit, len, distance) } - } - else { + } else { /* far away, but not yet in the another galaxy... */ - if (len < 7U) { - distance -= MAX_DISTANCE; - *op++ = (uint8_t)((len << 5U) + 31U); - *op++ = 255U; - *op++ = (uint8_t)(distance >> 8U); - *op++ = (uint8_t)(distance & 255U); - } - else { - distance -= MAX_DISTANCE; - *op++ = (7U << 5U) + 31U; - for (len -= 7U; len >= 255U; len -= 255U) - *op++ = 255U; - *op++ = (uint8_t)len; - *op++ = 255U; - *op++ = (uint8_t)(distance >> 8U); - *op++ = (uint8_t)(distance & 255U); + distance -= MAX_DISTANCE; + if (len < 7) { + MATCH_SHORT_FAR(op, op_limit, len, distance) + } else { + MATCH_LONG_FAR(op, op_limit, len, distance) } } /* update the hash at match boundary */ - if (ip < ip_limit) { - HASH_FUNCTION(hval, ip, hashlog); - htab[hval] = (uint16_t)(ip - ibase); + seq = BLOSCLZ_READU32(ip); + HASH_FUNCTION(hval, seq, hashlog) + htab[hval] = (uint32_t) (ip++ - ibase); + if (clevel == 9) { + // In some situations, including a second hash proves to be useful, + // but not in others. Activating here in max clevel only. + seq >>= 8U; + HASH_FUNCTION(hval, seq, hashlog) + htab[hval] = (uint32_t) (ip++ - ibase); } - ip += 2; + else { + ip++; + } + + if (BLOSCLZ_UNLIKELY(op + 1 > op_limit)) + goto out; + /* assuming literal copy */ *op++ = MAX_COPY - 1; - - // reset the number of max copies - nmax_copies = 0; } /* left-over as literal copy */ - ip_bound++; - while (ip <= ip_bound) { - if (BLOSCLZ_UNEXPECT_CONDITIONAL(op + 2 > op_limit)) goto out; + while (BLOSCLZ_UNLIKELY(ip <= ip_bound)) { + if (BLOSCLZ_UNLIKELY(op + 2 > op_limit)) goto out; *op++ = *ip++; copy++; - if (copy == MAX_COPY) { + if (BLOSCLZ_UNLIKELY(copy == MAX_COPY)) { copy = 0; *op++ = MAX_COPY - 1; } @@ -542,11 +606,10 @@ out: return 0; - } // See https://habr.com/en/company/yandex/blog/457612/ -#ifdef __AVX2__ +#if defined(__AVX2__) #if defined(_MSC_VER) #define ALIGNED_(x) __declspec(align(x)) @@ -564,23 +627,23 @@ static const ALIGNED_TYPE_(uint8_t, 16) masks[] = { - 0, 1, 2, 1, 4, 1, 4, 2, 8, 7, 6, 5, 4, 3, 2, 1, // offset = 0, not used as mask, but for shift - 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // offset = 1 - 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, - 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, - 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, - 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, - 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, - 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, - 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0, 1, 2, 3, 4, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0, 1, 2, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0, 1, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0, - 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // offset = 16 + 0, 1, 2, 1, 4, 1, 4, 2, 8, 7, 6, 5, 4, 3, 2, 1, // offset = 0, not used as mask, but for shift + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, // offset = 1 + 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, 0, 1, + 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2, 0, + 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 0, + 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, 4, 5, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 5, 6, 0, 1, 2, 3, 4, 5, 6, 0, 1, + 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 5, 6, 7, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 0, 1, 2, 3, 4, 5, 6, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 0, 1, 2, 3, 4, 5, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 0, 1, 2, 3, 4, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 0, 1, 2, 3, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 0, 1, 2, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 0, 1, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 0, + 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, // offset = 16 }; _mm_storeu_si128((__m128i *)(op), @@ -600,13 +663,123 @@ } #endif +// LZ4 wildCopy which can reach excellent copy bandwidth (even if insecure) +static inline void wild_copy(uint8_t *out, const uint8_t* from, uint8_t* end) { + uint8_t* d = out; + const uint8_t* s = from; + uint8_t* const e = end; -/** - Define blosc_decompress and blosc_decompress_unsafe. - */ -#define BLOSCLZ_SAFE -#include "blosclz_impl.inc" -#undef BLOSCLZ_SAFE -#define blosclz_decompress blosclz_decompress_unsafe -#include "blosclz_impl.inc" -#undef blosclz_decompress + do { memcpy(d,s,8); d+=8; s+=8; } while (d= 32) { + // match + int32_t len = (ctrl >> 5U) - 1 ; + int32_t ofs = (ctrl & 31U) << 8U; + uint8_t code; + const uint8_t* ref = op - ofs; + + if (len == 7 - 1) { + do { + if (BLOSCLZ_UNLIKELY(ip + 1 >= ip_limit)) { + return 0; + } + code = *ip++; + len += code; + } while (code == 255); + } + else { + if (BLOSCLZ_UNLIKELY(ip + 1 >= ip_limit)) { + return 0; + } + } + code = *ip++; + len += 3; + ref -= code; + + /* match from 16-bit distance */ + if (BLOSCLZ_UNLIKELY(code == 255)) { + if (ofs == (31U << 8U)) { + if (ip + 1 >= ip_limit) { + return 0; + } + ofs = (*ip++) << 8U; + ofs += *ip++; + ref = op - ofs - MAX_DISTANCE; + } + } + + if (BLOSCLZ_UNLIKELY(op + len > op_limit)) { + return 0; + } + + if (BLOSCLZ_UNLIKELY(ref - 1 < (uint8_t*)output)) { + return 0; + } + + if (BLOSCLZ_UNLIKELY(ip >= ip_limit)) break; + ctrl = *ip++; + + ref--; + if (ref == op - 1) { + /* optimized copy for a run */ + memset(op, *ref, len); + op += len; + } + else if ((op - ref >= 8) && (op_limit - op >= len + 8)) { + // copy with an overlap not larger than 8 + wild_copy(op, ref, op + len); + op += len; + } + else { + // general copy with any overlap +#if defined(__AVX2__) + if (op - ref <= 16) { + // This is not faster on a combination of compilers (clang, gcc, icc) or machines, but + // it is not slower either. Let's activate here for experimentation. + op = copy_match_16(op, ref, len); + } + else { +#endif + op = copy_match(op, ref, (unsigned) len); +#if defined(__AVX2__) + } +#endif + } + } + else { + // literal + ctrl++; + if (BLOSCLZ_UNLIKELY(op + ctrl > op_limit)) { + return 0; + } + if (BLOSCLZ_UNLIKELY(ip + ctrl > ip_limit)) { + return 0; + } + + memcpy(op, ip, ctrl); op += ctrl; ip += ctrl; + // On GCC-6, fastcopy this is still faster than plain memcpy + // However, using recent CLANG/LLVM 9.0, there is almost no difference + // in performance. + // And starting on CLANG/LLVM 10 and GCC 9, memcpy is generally faster. + // op = fastcopy(op, ip, (unsigned) ctrl); ip += ctrl; + + if (BLOSCLZ_UNLIKELY(ip >= ip_limit)) break; + ctrl = *ip++; + } + } + + return (int)(op - (uint8_t*)output); +} diff -Nru c-blosc-1.17.1+ds1/blosc/blosclz.h c-blosc-1.21.1+ds2/blosc/blosclz.h --- c-blosc-1.17.1+ds1/blosc/blosclz.h 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/blosc/blosclz.h 2021-10-06 08:16:29.000000000 +0000 @@ -3,7 +3,7 @@ Author: Francesc Alted - See LICENSES/BLOSC.txt for details about copyright and rights to use. + See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ /********************************************************************* @@ -20,6 +20,7 @@ extern "C" { #endif + /** Compress a block of data in the input buffer and returns the size of compressed block. The size of input buffer is specified by @@ -40,8 +41,8 @@ The input buffer and the output buffer can not overlap. */ -int blosclz_compress(const int opt_level, const void* input, int length, - void* output, int maxout, int shuffle); +int blosclz_compress(int opt_level, const void* input, int length, + void* output, int maxout, int split_block); /** Decompress a block of compressed data and returns the size of the @@ -57,13 +58,6 @@ int blosclz_decompress(const void* input, int length, void* output, int maxout); -/** - Same as above, except that it is not safe to run on invalid/untrusted input, - and may be slightly faster. - */ -int blosclz_decompress_unsafe(const void* input, int length, void* output, - int maxout); - #if defined (__cplusplus) } #endif diff -Nru c-blosc-1.17.1+ds1/blosc/blosclz_impl.inc c-blosc-1.21.1+ds2/blosc/blosclz_impl.inc --- c-blosc-1.17.1+ds1/blosc/blosclz_impl.inc 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/blosc/blosclz_impl.inc 1970-01-01 00:00:00.000000000 +0000 @@ -1,126 +0,0 @@ -/** - * Defines blosclz_decompress for a particular choice of BLOSCLZ_SAFE. - * - * This is included from blosclz.c once for each value of BLOSCLZ_SAFE. - */ - -int blosclz_decompress(const void* input, int length, void* output, int maxout) { - const uint8_t* ip = (const uint8_t*)input; - const uint8_t* ip_limit = ip + length; - uint8_t* op = (uint8_t*)output; - int32_t ctrl; - int32_t loop = 1; -#ifdef BLOSCLZ_SAFE - uint8_t* op_limit = op + maxout; - if (BLOSCLZ_UNEXPECT_CONDITIONAL(length == 0)) { - return 0; - } -#endif - ctrl = (*ip++) & 31; - - do { - uint8_t* ref = op; - int32_t len = ctrl >> 5; - int32_t ofs = (ctrl & 31) << 8; - - if (ctrl >= 32) { - uint8_t code; - len--; - ref -= ofs; - if (len == 7 - 1) { - do { -#ifdef BLOSCLZ_SAFE - if (BLOSCLZ_UNEXPECT_CONDITIONAL(ip + 1 >= ip_limit)) { - return 0; - } -#endif - code = *ip++; - len += code; - } while (code == 255); - } else { -#ifdef BLOSCLZ_SAFE - if (BLOSCLZ_UNEXPECT_CONDITIONAL(ip >= ip_limit)) { - return 0; - } -#endif - } - code = *ip++; - ref -= code; - - /* match from 16-bit distance */ - if (BLOSCLZ_UNEXPECT_CONDITIONAL(code == 255)) if (BLOSCLZ_EXPECT_CONDITIONAL(ofs == (31 << 8))) { -#ifdef BLOSCLZ_SAFE - if (BLOSCLZ_UNEXPECT_CONDITIONAL(ip + 1 >= ip_limit)) { - return 0; - } -#endif - ofs = (*ip++) << 8; - ofs += *ip++; - ref = op - ofs - MAX_DISTANCE; - } - -#ifdef BLOSCLZ_SAFE - if (BLOSCLZ_UNEXPECT_CONDITIONAL(op + len + 3 > op_limit)) { - return 0; - } - - if (BLOSCLZ_UNEXPECT_CONDITIONAL(ref - 1 < (uint8_t*)output)) { - return 0; - } -#endif - - if (BLOSCLZ_EXPECT_CONDITIONAL(ip < ip_limit)) - ctrl = *ip++; - else - loop = 0; - - if (ref == op) { - /* optimized copy for a run */ - uint8_t b = ref[-1]; - memset(op, b, len + 3); - op += len + 3; - } - else { - /* copy from reference */ - ref--; - len += 3; -#ifdef __AVX2__ - if (op - ref <= 16) { - // This is not faster on a combination of compilers (clang, gcc, icc) or machines, but - // it is not slower either. Let's activate here for experimentation. - op = copy_match_16(op, ref, len); - } - else { -#endif - // We absolutely need a blosc_internal_copy_match here - op = blosc_internal_copy_match(op, ref, (unsigned) len); -#ifdef __AVX2__ - } -#endif - } - } - else { - ctrl++; -#ifdef BLOSCLZ_SAFE - if (BLOSCLZ_UNEXPECT_CONDITIONAL(op + ctrl > op_limit)) { - return 0; - } - if (BLOSCLZ_UNEXPECT_CONDITIONAL(ip + ctrl > ip_limit)) { - return 0; - } -#endif - - // memcpy(op, ip, ctrl); op += ctrl; ip += ctrl; - // On GCC-6, blosc_internal_fastcopy this is still faster than plain memcpy - // However, using recent CLANG/LLVM 9.0, there is almost no difference - // in performance. - op = blosc_internal_fastcopy(op, ip, (unsigned) ctrl); ip += ctrl; - - loop = (int32_t)BLOSCLZ_EXPECT_CONDITIONAL(ip < ip_limit); - if (loop) - ctrl = *ip++; - } - } while (BLOSCLZ_EXPECT_CONDITIONAL(loop)); - - return (int)(op - (uint8_t*)output); -} diff -Nru c-blosc-1.17.1+ds1/blosc/CMakeLists.txt c-blosc-1.21.1+ds2/blosc/CMakeLists.txt --- c-blosc-1.17.1+ds1/blosc/CMakeLists.txt 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/blosc/CMakeLists.txt 2021-10-06 08:16:29.000000000 +0000 @@ -14,7 +14,7 @@ if (LZ4_FOUND) set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${LZ4_INCLUDE_DIR}) else(LZ4_FOUND) - set(LZ4_LOCAL_DIR ${INTERNAL_LIBS}/lz4-1.9.2) + set(LZ4_LOCAL_DIR ${INTERNAL_LIBS}/lz4-1.9.3) set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${LZ4_LOCAL_DIR}) endif(LZ4_FOUND) endif(NOT DEACTIVATE_LZ4) @@ -32,7 +32,7 @@ if (ZLIB_FOUND) set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZLIB_INCLUDE_DIR}) else(ZLIB_FOUND) - set(ZLIB_LOCAL_DIR ${INTERNAL_LIBS}/zlib-1.2.8) + set(ZLIB_LOCAL_DIR ${INTERNAL_LIBS}/zlib-1.2.11) set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZLIB_LOCAL_DIR}) endif(ZLIB_FOUND) endif(NOT DEACTIVATE_ZLIB) @@ -41,7 +41,7 @@ if (ZSTD_FOUND) set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZSTD_INCLUDE_DIR}) else (ZSTD_FOUND) - set(ZSTD_LOCAL_DIR ${INTERNAL_LIBS}/zstd-1.4.4) + set(ZSTD_LOCAL_DIR ${INTERNAL_LIBS}/zstd-1.5.0) set(BLOSC_INCLUDE_DIRS ${BLOSC_INCLUDE_DIRS} ${ZSTD_LOCAL_DIR} ${ZSTD_LOCAL_DIR}/common) endif (ZSTD_FOUND) endif (NOT DEACTIVATE_ZSTD) @@ -65,7 +65,8 @@ set(lib_dir lib${LIB_SUFFIX}) set(version_string ${BLOSC_VERSION_MAJOR}.${BLOSC_VERSION_MINOR}.${BLOSC_VERSION_PATCH}) -set(CMAKE_THREAD_PREFER_PTHREAD TRUE) +set(CMAKE_THREAD_PREFER_PTHREAD TRUE) # pre 3.1 +set(THREADS_PREFER_PTHREAD_FLAG TRUE) # CMake 3.1+ if(WIN32) # try to use the system library find_package(Threads) @@ -73,11 +74,19 @@ message(STATUS "using the internal pthread library for win32 systems.") set(SOURCES ${SOURCES} win32/pthread.c) else(NOT Threads_FOUND) - set(LIBS ${LIBS} ${CMAKE_THREAD_LIBS_INIT}) + if(CMAKE_VERSION VERSION_LESS 3.1) + set(LIBS ${LIBS} ${CMAKE_THREAD_LIBS_INIT}) + else() + set(LIBS ${LIBS} Threads::Threads) + endif() endif(NOT Threads_FOUND) else(WIN32) find_package(Threads REQUIRED) - set(LIBS ${LIBS} ${CMAKE_THREAD_LIBS_INIT}) + if(CMAKE_VERSION VERSION_LESS 3.1) + set(LIBS ${LIBS} ${CMAKE_THREAD_LIBS_INIT}) + else() + set(LIBS ${LIBS} Threads::Threads) + endif() endif(WIN32) if(NOT DEACTIVATE_LZ4) @@ -86,6 +95,7 @@ else(LZ4_FOUND) file(GLOB LZ4_FILES ${LZ4_LOCAL_DIR}/*.c) set(SOURCES ${SOURCES} ${LZ4_FILES}) + source_group("LZ4" FILES ${LZ4_FILES}) endif(LZ4_FOUND) endif(NOT DEACTIVATE_LZ4) @@ -95,6 +105,7 @@ else(SNAPPY_FOUND) file(GLOB SNAPPY_FILES ${SNAPPY_LOCAL_DIR}/*.cc) set(SOURCES ${SOURCES} ${SNAPPY_FILES}) + source_group("Snappy" FILES ${SNAPPY_FILES}) endif(SNAPPY_FOUND) endif(NOT DEACTIVATE_SNAPPY) @@ -104,6 +115,7 @@ else(ZLIB_FOUND) file(GLOB ZLIB_FILES ${ZLIB_LOCAL_DIR}/*.c) set(SOURCES ${SOURCES} ${ZLIB_FILES}) + source_group("Zlib" FILES ${ZLIB_FILES}) endif(ZLIB_FOUND) endif(NOT DEACTIVATE_ZLIB) @@ -116,6 +128,7 @@ ${ZSTD_LOCAL_DIR}/compress/*.c ${ZSTD_LOCAL_DIR}/decompress/*.c) set(SOURCES ${SOURCES} ${ZSTD_FILES}) + source_group("Zstd" FILES ${ZSTD_FILES}) endif (ZSTD_FOUND) endif (NOT DEACTIVATE_ZSTD) @@ -182,13 +195,6 @@ set_property( TARGET blosc_shared_testing APPEND PROPERTY COMPILE_DEFINITIONS BLOSC_TESTING) - # TEMP : CMake doesn't automatically add -lpthread here like it does - # for the blosc_shared target. Force it for now. - if(UNIX) - set_property( - TARGET blosc_shared_testing - APPEND PROPERTY LINK_FLAGS "-lpthread") - endif() endif() if (BUILD_SHARED) diff -Nru c-blosc-1.17.1+ds1/blosc/fastcopy.c c-blosc-1.21.1+ds2/blosc/fastcopy.c --- c-blosc-1.17.1+ds1/blosc/fastcopy.c 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/blosc/fastcopy.c 2021-10-06 08:16:29.000000000 +0000 @@ -112,7 +112,7 @@ __m256i chunk; chunk = _mm256_loadu_si256((__m256i*)from); _mm256_storeu_si256((__m256i*)out, chunk); - from += 32; out += 32; + out += 32; #elif defined(__SSE2__) __m128i chunk; chunk = _mm_loadu_si128((__m128i*)from); @@ -139,14 +139,15 @@ return out; } -#if defined(__AVX2__) -static inline unsigned char *copy_32_bytes_aligned(unsigned char *out, const unsigned char *from) { - __m256i chunk; - chunk = _mm256_load_si256((__m256i*)from); - _mm256_storeu_si256((__m256i*)out, chunk); - return out + 32; -} -#endif // __AVX2__ +// This is never used, so comment it out +//#if defined(__AVX2__) +//static inline unsigned char *copy_32_bytes_aligned(unsigned char *out, const unsigned char *from) { +// __m256i chunk; +// chunk = _mm256_load_si256((__m256i*)from); +// _mm256_storeu_si256((__m256i*)out, chunk); +// return out + 32; +//} +//#endif // __AVX2__ /* Copy LEN bytes (7 or fewer) from FROM into OUT. Return OUT + LEN. */ static inline unsigned char *copy_bytes(unsigned char *out, const unsigned char *from, unsigned len) { @@ -208,45 +209,45 @@ case 7: out = copy_8_bytes(out, from); from += sz; - #ifdef AVOID_FALLTHROUGH_WARNING + #ifdef AVOID_FALLTHROUGH_WARNING __attribute__ ((fallthrough)); // Shut-up -Wimplicit-fallthrough warning in GCC - #endif + #endif case 6: out = copy_8_bytes(out, from); from += sz; - #ifdef AVOID_FALLTHROUGH_WARNING + #ifdef AVOID_FALLTHROUGH_WARNING __attribute__ ((fallthrough)); - #endif + #endif case 5: out = copy_8_bytes(out, from); from += sz; - #ifdef AVOID_FALLTHROUGH_WARNING + #ifdef AVOID_FALLTHROUGH_WARNING __attribute__ ((fallthrough)); - #endif + #endif case 4: out = copy_8_bytes(out, from); from += sz; - #ifdef AVOID_FALLTHROUGH_WARNING + #ifdef AVOID_FALLTHROUGH_WARNING __attribute__ ((fallthrough)); - #endif + #endif case 3: out = copy_8_bytes(out, from); from += sz; - #ifdef AVOID_FALLTHROUGH_WARNING + #ifdef AVOID_FALLTHROUGH_WARNING __attribute__ ((fallthrough)); - #endif + #endif case 2: out = copy_8_bytes(out, from); from += sz; - #ifdef AVOID_FALLTHROUGH_WARNING + #ifdef AVOID_FALLTHROUGH_WARNING __attribute__ ((fallthrough)); - #endif + #endif case 1: out = copy_8_bytes(out, from); from += sz; - #ifdef AVOID_FALLTHROUGH_WARNING + #ifdef AVOID_FALLTHROUGH_WARNING __attribute__ ((fallthrough)); - #endif + #endif default: break; } @@ -492,7 +493,7 @@ /* Byte by byte semantics: copy LEN bytes from FROM and write them to OUT. Return OUT + LEN. */ -unsigned char *blosc_internal_fastcopy(unsigned char *out, const unsigned char *from, unsigned len) { +unsigned char *fastcopy(unsigned char *out, const unsigned char *from, unsigned len) { switch (len) { case 32: return copy_32_bytes(out, from); @@ -525,7 +526,7 @@ /* Copy a run */ -unsigned char* blosc_internal_copy_match(unsigned char *out, const unsigned char *from, unsigned len) { +unsigned char* copy_match(unsigned char *out, const unsigned char *from, unsigned len) { #if defined(__AVX2__) unsigned sz = sizeof(__m256i); #elif defined(__SSE2__) @@ -534,12 +535,6 @@ unsigned sz = sizeof(uint64_t); #endif - // If out and from are away more than the size of the copy, then a blosc_internal_fastcopy is safe - unsigned overlap_dist = (unsigned) (out - from); - if (overlap_dist > sz) { - return blosc_internal_fastcopy(out, from, len); - } - #if ((defined(__GNUC__) && BLOSC_GCC_VERSION < 800) && !defined(__clang__) && !defined(__ICC) && !defined(__ICL)) // GCC < 8 in fully optimization mode seems to have problems with the code further below so stop here for (; len > 0; len--) { @@ -548,6 +543,12 @@ return out; #endif + // If out and from are away more than the size of the copy, then a fastcopy is safe + unsigned overlap_dist = (unsigned) (out - from); + if (overlap_dist > sz) { + return fastcopy(out, from, len); + } + // Otherwise we need to be more careful so as not to overwrite destination switch (overlap_dist) { case 32: diff -Nru c-blosc-1.17.1+ds1/blosc/fastcopy.h c-blosc-1.21.1+ds2/blosc/fastcopy.h --- c-blosc-1.17.1+ds1/blosc/fastcopy.h 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/blosc/fastcopy.h 2021-10-06 08:16:29.000000000 +0000 @@ -4,16 +4,16 @@ Author: Francesc Alted Creation date: 2018-01-03 - See LICENSES/BLOSC.txt for details about copyright and rights to use. + See LICENSE.txt for details about copyright and rights to use. **********************************************************************/ #ifndef BLOSC_FASTCOPY_H #define BLOSC_FASTCOPY_H /* Same semantics than memcpy() */ -unsigned char *blosc_internal_fastcopy(unsigned char *out, const unsigned char *from, unsigned len); +unsigned char *fastcopy(unsigned char *out, const unsigned char *from, unsigned len); -/* Same as blosc_internal_fastcopy() but without overwriting origin or destination when they overlap */ -unsigned char* blosc_internal_copy_match(unsigned char *out, const unsigned char *from, unsigned len); +/* Same as fastcopy() but without overwriting origin or destination when they overlap */ +unsigned char* copy_match(unsigned char *out, const unsigned char *from, unsigned len); -#endif /*BLOSC_FASTCOPY_H*/ +#endif //BLOSC_FASTCOPY_H diff -Nru c-blosc-1.17.1+ds1/blosc/shuffle.c c-blosc-1.21.1+ds2/blosc/shuffle.c --- c-blosc-1.17.1+ds1/blosc/shuffle.c 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/blosc/shuffle.c 2021-10-06 08:16:29.000000000 +0000 @@ -8,6 +8,7 @@ **********************************************************************/ #include "shuffle.h" +#include "blosc-common.h" #include "shuffle-generic.h" #include "bitshuffle-generic.h" #include "blosc-comp-features.h" @@ -173,6 +174,8 @@ #define _XCR_XFEATURE_ENABLED_MASK 0 +#if !(defined(_IMMINTRIN_H_INCLUDED) && (BLOSC_GCC_VERSION >= 900)) + /* Reads the content of an extended control register. https://software.intel.com/en-us/articles/how-to-detect-new-instruction-support-in-the-4th-generation-intel-core-processor-family */ @@ -192,7 +195,12 @@ return ((uint64_t)edx << 32) | eax; } -#endif /* defined(_MSC_FULL_VER) */ +#else + +#define blosc_internal_xgetbv _xgetbv + +#endif // !(defined(_IMMINTRIN_H_INCLUDED) && (BLOSC_GCC_VERSION >= 900)) +#endif /* defined(_MSC_FULL_VER) */ #ifndef _XCR_XFEATURE_ENABLED_MASK #define _XCR_XFEATURE_ENABLED_MASK 0x0 @@ -404,14 +412,20 @@ /* Initialize the shuffle implementation if necessary. */ init_shuffle_implementation(); - if ((size % 8) == 0) + if ((size % 8) == 0) { /* The number of elems is a multiple of 8 which is supported by bitshuffle. */ - return (int)(host_implementation.bitshuffle)((void*)_src, (void*)_dest, - blocksize / bytesoftype, - bytesoftype, (void*)_tmp); - else - memcpy((void*)_dest, (void*)_src, blocksize); + int ret = (int)(host_implementation.bitshuffle)((void *) _src, (void *) _dest, + blocksize / bytesoftype, + bytesoftype, (void *) _tmp); + /* Copy the leftovers */ + size_t offset = size * bytesoftype; + memcpy((void *) (_dest + offset), (void *) (_src + offset), blocksize - offset); + return ret; + } + else { + memcpy((void *) _dest, (void *) _src, blocksize); + } return size; } @@ -425,13 +439,19 @@ /* Initialize the shuffle implementation if necessary. */ init_shuffle_implementation(); - if ((size % 8) == 0) + if ((size % 8) == 0) { /* The number of elems is a multiple of 8 which is supported by bitshuffle. */ - return (int)(host_implementation.bitunshuffle)((void*)_src, (void*)_dest, - blocksize / bytesoftype, - bytesoftype, (void*)_tmp); - else - memcpy((void*)_dest, (void*)_src, blocksize); + int ret = (int) (host_implementation.bitunshuffle)((void *) _src, (void *) _dest, + blocksize / bytesoftype, + bytesoftype, (void *) _tmp); + /* Copy the leftovers */ + size_t offset = size * bytesoftype; + memcpy((void *) (_dest + offset), (void *) (_src + offset), blocksize - offset); + return ret; + } + else { + memcpy((void *) _dest, (void *) _src, blocksize); + } return size; } diff -Nru c-blosc-1.17.1+ds1/blosc/win32/pthread.c c-blosc-1.21.1+ds2/blosc/win32/pthread.c --- c-blosc-1.17.1+ds1/blosc/win32/pthread.c 1970-01-01 00:00:00.000000000 +0000 +++ c-blosc-1.21.1+ds2/blosc/win32/pthread.c 2021-10-06 08:16:29.000000000 +0000 @@ -0,0 +1,218 @@ +/* + * Code for simulating pthreads API on Windows. This is Git-specific, + * but it is enough for Numexpr needs too. + * + * Copyright (C) 2009 Andrzej K. Haczewski + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * DISCLAIMER: The implementation is Git-specific, it is subset of original + * Pthreads API, without lots of other features that Git doesn't use. + * Git also makes sure that the passed arguments are valid, so there's + * no need for double-checking. + */ + +#include "pthread.h" + +#include +#include +#include +#include +#include + + +void die(const char *err, ...) +{ + printf("%s", err); + exit(-1); +} + +static unsigned __stdcall win32_start_routine(void *arg) +{ + pthread_t *thread = (pthread_t*)arg; + thread->arg = thread->start_routine(thread->arg); + return 0; +} + +int pthread_create(pthread_t *thread, const void *unused, + void *(*start_routine)(void*), void *arg) +{ + thread->arg = arg; + thread->start_routine = start_routine; + thread->handle = (HANDLE) + _beginthreadex(NULL, 0, win32_start_routine, thread, 0, NULL); + + if (!thread->handle) + return errno; + else + return 0; +} + +int win32_pthread_join(pthread_t *thread, void **value_ptr) +{ + DWORD result = WaitForSingleObject(thread->handle, INFINITE); + switch (result) { + case WAIT_OBJECT_0: + if (value_ptr) + *value_ptr = thread->arg; + return 0; + case WAIT_ABANDONED: + return EINVAL; + default: + return GetLastError(); + } +} + +int pthread_cond_init(pthread_cond_t *cond, const void *unused) +{ + cond->waiters = 0; + cond->was_broadcast = 0; + InitializeCriticalSection(&cond->waiters_lock); + + cond->sema = CreateSemaphore(NULL, 0, LONG_MAX, NULL); + if (!cond->sema) + die("CreateSemaphore() failed"); + + cond->continue_broadcast = CreateEvent(NULL, /* security */ + FALSE, /* auto-reset */ + FALSE, /* not signaled */ + NULL); /* name */ + if (!cond->continue_broadcast) + die("CreateEvent() failed"); + + return 0; +} + +int pthread_cond_destroy(pthread_cond_t *cond) +{ + CloseHandle(cond->sema); + CloseHandle(cond->continue_broadcast); + DeleteCriticalSection(&cond->waiters_lock); + return 0; +} + +int pthread_cond_wait(pthread_cond_t *cond, CRITICAL_SECTION *mutex) +{ + int last_waiter; + + EnterCriticalSection(&cond->waiters_lock); + cond->waiters++; + LeaveCriticalSection(&cond->waiters_lock); + + /* + * Unlock external mutex and wait for signal. + * NOTE: we've held mutex locked long enough to increment + * waiters count above, so there's no problem with + * leaving mutex unlocked before we wait on semaphore. + */ + LeaveCriticalSection(mutex); + + /* let's wait - ignore return value */ + WaitForSingleObject(cond->sema, INFINITE); + + /* + * Decrease waiters count. If we are the last waiter, then we must + * notify the broadcasting thread that it can continue. + * But if we continued due to cond_signal, we do not have to do that + * because the signaling thread knows that only one waiter continued. + */ + EnterCriticalSection(&cond->waiters_lock); + cond->waiters--; + last_waiter = cond->was_broadcast && cond->waiters == 0; + LeaveCriticalSection(&cond->waiters_lock); + + if (last_waiter) { + /* + * cond_broadcast was issued while mutex was held. This means + * that all other waiters have continued, but are contending + * for the mutex at the end of this function because the + * broadcasting thread did not leave cond_broadcast, yet. + * (This is so that it can be sure that each waiter has + * consumed exactly one slice of the semaphore.) + * The last waiter must tell the broadcasting thread that it + * can go on. + */ + SetEvent(cond->continue_broadcast); + /* + * Now we go on to contend with all other waiters for + * the mutex. Auf in den Kampf! + */ + } + /* lock external mutex again */ + EnterCriticalSection(mutex); + + return 0; +} + +/* + * IMPORTANT: This implementation requires that pthread_cond_signal + * is called while the mutex is held that is used in the corresponding + * pthread_cond_wait calls! + */ +int pthread_cond_signal(pthread_cond_t *cond) +{ + int have_waiters; + + EnterCriticalSection(&cond->waiters_lock); + have_waiters = cond->waiters > 0; + LeaveCriticalSection(&cond->waiters_lock); + + /* + * Signal only when there are waiters + */ + if (have_waiters) + return ReleaseSemaphore(cond->sema, 1, NULL) ? + 0 : GetLastError(); + else + return 0; +} + +/* + * DOUBLY IMPORTANT: This implementation requires that pthread_cond_broadcast + * is called while the mutex is held that is used in the corresponding + * pthread_cond_wait calls! + */ +int pthread_cond_broadcast(pthread_cond_t *cond) +{ + EnterCriticalSection(&cond->waiters_lock); + + if ((cond->was_broadcast = cond->waiters > 0)) { + /* wake up all waiters */ + ReleaseSemaphore(cond->sema, cond->waiters, NULL); + LeaveCriticalSection(&cond->waiters_lock); + /* + * At this point all waiters continue. Each one takes its + * slice of the semaphore. Now it's our turn to wait: Since + * the external mutex is held, no thread can leave cond_wait, + * yet. For this reason, we can be sure that no thread gets + * a chance to eat *more* than one slice. OTOH, it means + * that the last waiter must send us a wake-up. + */ + WaitForSingleObject(cond->continue_broadcast, INFINITE); + /* + * Since the external mutex is held, no thread can enter + * cond_wait, and, hence, it is safe to reset this flag + * without cond->waiters_lock held. + */ + cond->was_broadcast = 0; + } else { + LeaveCriticalSection(&cond->waiters_lock); + } + return 0; +} diff -Nru c-blosc-1.17.1+ds1/blosc/win32/pthread.h c-blosc-1.21.1+ds2/blosc/win32/pthread.h --- c-blosc-1.17.1+ds1/blosc/win32/pthread.h 1970-01-01 00:00:00.000000000 +0000 +++ c-blosc-1.21.1+ds2/blosc/win32/pthread.h 2021-10-06 08:16:29.000000000 +0000 @@ -0,0 +1,112 @@ +/* + * Code for simulating pthreads API on Windows. This is Git-specific, + * but it is enough for Numexpr needs too. + * + * Copyright (C) 2009 Andrzej K. Haczewski + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN + * THE SOFTWARE. + * + * DISCLAIMER: The implementation is Git-specific, it is subset of original + * Pthreads API, without lots of other features that Git doesn't use. + * Git also makes sure that the passed arguments are valid, so there's + * no need for double-checking. + */ + +#ifndef PTHREAD_H +#define PTHREAD_H + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif + +#include + +/* + * Defines that adapt Windows API threads to pthreads API + */ +#define pthread_mutex_t CRITICAL_SECTION + +#define pthread_mutex_init(a,b) InitializeCriticalSection((a)) +#define pthread_mutex_destroy(a) DeleteCriticalSection((a)) +#define pthread_mutex_lock EnterCriticalSection +#define pthread_mutex_unlock LeaveCriticalSection + +/* + * Implement simple condition variable for Windows threads, based on ACE + * implementation. + * + * See original implementation: http://bit.ly/1vkDjo + * ACE homepage: http://www.cse.wustl.edu/~schmidt/ACE.html + * See also: http://www.cse.wustl.edu/~schmidt/win32-cv-1.html + */ +typedef struct { + LONG waiters; + int was_broadcast; + CRITICAL_SECTION waiters_lock; + HANDLE sema; + HANDLE continue_broadcast; +} pthread_cond_t; + +extern int pthread_cond_init(pthread_cond_t *cond, const void *unused); +extern int pthread_cond_destroy(pthread_cond_t *cond); +extern int pthread_cond_wait(pthread_cond_t *cond, CRITICAL_SECTION *mutex); +extern int pthread_cond_signal(pthread_cond_t *cond); +extern int pthread_cond_broadcast(pthread_cond_t *cond); + +/* + * Simple thread creation implementation using pthread API + */ +typedef struct { + HANDLE handle; + void *(*start_routine)(void*); + void *arg; +} pthread_t; + +extern int pthread_create(pthread_t *thread, const void *unused, + void *(*start_routine)(void*), void *arg); + +/* + * To avoid the need of copying a struct, we use small macro wrapper to pass + * pointer to win32_pthread_join instead. + */ +#define pthread_join(a, b) win32_pthread_join(&(a), (b)) + +extern int win32_pthread_join(pthread_t *thread, void **value_ptr); + +/** + * pthread_once implementation based on the MS Windows One-Time Initialization + * (https://docs.microsoft.com/en-us/windows/desktop/Sync/one-time-initialization) + * APIs. + */ +typedef INIT_ONCE pthread_once_t; +#define PTHREAD_ONCE_INIT INIT_ONCE_STATIC_INIT +#define pthread_once blosc_internal_pthread_once /* Avoid symbol conflicts */ +static int blosc_internal_pthread_once(pthread_once_t* once_control, + void (*init_routine)(void)) { + BOOL pending; + InitOnceBeginInitialize(once_control, /*dwFlags=*/0, /*fPending=*/&pending, + NULL); + if (pending == TRUE) { + init_routine(); + InitOnceComplete(once_control, /*dwFlags=*/0, /*lpContext=*/NULL); + } + return 0; +} + +#endif /* PTHREAD_H */ diff -Nru c-blosc-1.17.1+ds1/blosc/win32/stdint-windows.h c-blosc-1.21.1+ds2/blosc/win32/stdint-windows.h --- c-blosc-1.17.1+ds1/blosc/win32/stdint-windows.h 1970-01-01 00:00:00.000000000 +0000 +++ c-blosc-1.21.1+ds2/blosc/win32/stdint-windows.h 2021-10-06 08:16:29.000000000 +0000 @@ -0,0 +1,259 @@ +// ISO C9x compliant stdint.h for Microsoft Visual Studio +// Based on ISO/IEC 9899:TC2 Committee draft (May 6, 2005) WG14/N1124 +// +// Copyright (c) 2006-2013 Alexander Chemeris +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are met: +// +// 1. Redistributions of source code must retain the above copyright notice, +// this list of conditions and the following disclaimer. +// +// 2. Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// 3. Neither the name of the product nor the names of its contributors may +// be used to endorse or promote products derived from this software +// without specific prior written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR IMPLIED +// WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF +// MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO +// EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; +// OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, +// WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR +// OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF +// ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +/////////////////////////////////////////////////////////////////////////////// + +#ifndef _MSC_VER // [ +#error "Use this header only with Microsoft Visual C++ compilers!" +#endif // _MSC_VER ] + +#ifndef _MSC_STDINT_H_ // [ +#define _MSC_STDINT_H_ + +#if _MSC_VER > 1000 +#pragma once +#endif + +#if _MSC_VER >= 1600 // [ +#include +#else // ] _MSC_VER >= 1600 [ + +#include + +// For Visual Studio 6 in C++ mode and for many Visual Studio versions when +// compiling for ARM we should wrap include with 'extern "C++" {}' +// or compiler give many errors like this: +// error C2733: second C linkage of overloaded function 'wmemchr' not allowed +#ifdef __cplusplus +extern "C" { +#endif +# include +#ifdef __cplusplus +} +#endif + +// Define _W64 macros to mark types changing their size, like intptr_t. +#ifndef _W64 +# if !defined(__midl) && (defined(_X86_) || defined(_M_IX86)) && _MSC_VER >= 1300 +# define _W64 __w64 +# else +# define _W64 +# endif +#endif + + +// 7.18.1 Integer types + +// 7.18.1.1 Exact-width integer types + +// Visual Studio 6 and Embedded Visual C++ 4 doesn't +// realize that, e.g. char has the same size as __int8 +// so we give up on __intX for them. +#if (_MSC_VER < 1300) + typedef signed char int8_t; + typedef signed short int16_t; + typedef signed int int32_t; + typedef unsigned char uint8_t; + typedef unsigned short uint16_t; + typedef unsigned int uint32_t; +#else + typedef signed __int8 int8_t; + typedef signed __int16 int16_t; + typedef signed __int32 int32_t; + typedef unsigned __int8 uint8_t; + typedef unsigned __int16 uint16_t; + typedef unsigned __int32 uint32_t; +#endif +typedef signed __int64 int64_t; +typedef unsigned __int64 uint64_t; + + +// 7.18.1.2 Minimum-width integer types +typedef int8_t int_least8_t; +typedef int16_t int_least16_t; +typedef int32_t int_least32_t; +typedef int64_t int_least64_t; +typedef uint8_t uint_least8_t; +typedef uint16_t uint_least16_t; +typedef uint32_t uint_least32_t; +typedef uint64_t uint_least64_t; + +// 7.18.1.3 Fastest minimum-width integer types +typedef int8_t int_fast8_t; +typedef int16_t int_fast16_t; +typedef int32_t int_fast32_t; +typedef int64_t int_fast64_t; +typedef uint8_t uint_fast8_t; +typedef uint16_t uint_fast16_t; +typedef uint32_t uint_fast32_t; +typedef uint64_t uint_fast64_t; + +// 7.18.1.4 Integer types capable of holding object pointers +#ifdef _WIN64 // [ + typedef signed __int64 intptr_t; + typedef unsigned __int64 uintptr_t; +#else // _WIN64 ][ + typedef _W64 signed int intptr_t; + typedef _W64 unsigned int uintptr_t; +#endif // _WIN64 ] + +// 7.18.1.5 Greatest-width integer types +typedef int64_t intmax_t; +typedef uint64_t uintmax_t; + + +// 7.18.2 Limits of specified-width integer types + +#if !defined(__cplusplus) || defined(__STDC_LIMIT_MACROS) // [ See footnote 220 at page 257 and footnote 221 at page 259 + +// 7.18.2.1 Limits of exact-width integer types +#define INT8_MIN ((int8_t)_I8_MIN) +#define INT8_MAX _I8_MAX +#define INT16_MIN ((int16_t)_I16_MIN) +#define INT16_MAX _I16_MAX +#define INT32_MIN ((int32_t)_I32_MIN) +#define INT32_MAX _I32_MAX +#define INT64_MIN ((int64_t)_I64_MIN) +#define INT64_MAX _I64_MAX +#define UINT8_MAX _UI8_MAX +#define UINT16_MAX _UI16_MAX +#define UINT32_MAX _UI32_MAX +#define UINT64_MAX _UI64_MAX + +// 7.18.2.2 Limits of minimum-width integer types +#define INT_LEAST8_MIN INT8_MIN +#define INT_LEAST8_MAX INT8_MAX +#define INT_LEAST16_MIN INT16_MIN +#define INT_LEAST16_MAX INT16_MAX +#define INT_LEAST32_MIN INT32_MIN +#define INT_LEAST32_MAX INT32_MAX +#define INT_LEAST64_MIN INT64_MIN +#define INT_LEAST64_MAX INT64_MAX +#define UINT_LEAST8_MAX UINT8_MAX +#define UINT_LEAST16_MAX UINT16_MAX +#define UINT_LEAST32_MAX UINT32_MAX +#define UINT_LEAST64_MAX UINT64_MAX + +// 7.18.2.3 Limits of fastest minimum-width integer types +#define INT_FAST8_MIN INT8_MIN +#define INT_FAST8_MAX INT8_MAX +#define INT_FAST16_MIN INT16_MIN +#define INT_FAST16_MAX INT16_MAX +#define INT_FAST32_MIN INT32_MIN +#define INT_FAST32_MAX INT32_MAX +#define INT_FAST64_MIN INT64_MIN +#define INT_FAST64_MAX INT64_MAX +#define UINT_FAST8_MAX UINT8_MAX +#define UINT_FAST16_MAX UINT16_MAX +#define UINT_FAST32_MAX UINT32_MAX +#define UINT_FAST64_MAX UINT64_MAX + +// 7.18.2.4 Limits of integer types capable of holding object pointers +#ifdef _WIN64 // [ +# define INTPTR_MIN INT64_MIN +# define INTPTR_MAX INT64_MAX +# define UINTPTR_MAX UINT64_MAX +#else // _WIN64 ][ +# define INTPTR_MIN INT32_MIN +# define INTPTR_MAX INT32_MAX +# define UINTPTR_MAX UINT32_MAX +#endif // _WIN64 ] + +// 7.18.2.5 Limits of greatest-width integer types +#define INTMAX_MIN INT64_MIN +#define INTMAX_MAX INT64_MAX +#define UINTMAX_MAX UINT64_MAX + +// 7.18.3 Limits of other integer types + +#ifdef _WIN64 // [ +# define PTRDIFF_MIN _I64_MIN +# define PTRDIFF_MAX _I64_MAX +#else // _WIN64 ][ +# define PTRDIFF_MIN _I32_MIN +# define PTRDIFF_MAX _I32_MAX +#endif // _WIN64 ] + +#define SIG_ATOMIC_MIN INT_MIN +#define SIG_ATOMIC_MAX INT_MAX + +#ifndef SIZE_MAX // [ +# ifdef _WIN64 // [ +# define SIZE_MAX _UI64_MAX +# else // _WIN64 ][ +# define SIZE_MAX _UI32_MAX +# endif // _WIN64 ] +#endif // SIZE_MAX ] + +// WCHAR_MIN and WCHAR_MAX are also defined in +#ifndef WCHAR_MIN // [ +# define WCHAR_MIN 0 +#endif // WCHAR_MIN ] +#ifndef WCHAR_MAX // [ +# define WCHAR_MAX _UI16_MAX +#endif // WCHAR_MAX ] + +#define WINT_MIN 0 +#define WINT_MAX _UI16_MAX + +#endif // __STDC_LIMIT_MACROS ] + + +// 7.18.4 Limits of other integer types + +#if !defined(__cplusplus) || defined(__STDC_CONSTANT_MACROS) // [ See footnote 224 at page 260 + +// 7.18.4.1 Macros for minimum-width integer constants + +#define INT8_C(val) val##i8 +#define INT16_C(val) val##i16 +#define INT32_C(val) val##i32 +#define INT64_C(val) val##i64 + +#define UINT8_C(val) val##ui8 +#define UINT16_C(val) val##ui16 +#define UINT32_C(val) val##ui32 +#define UINT64_C(val) val##ui64 + +// 7.18.4.2 Macros for greatest-width integer constants +// These #ifndef's are needed to prevent collisions with . +// Check out Issue 9 for the details. +#ifndef INTMAX_C // [ +# define INTMAX_C INT64_C +#endif // INTMAX_C ] +#ifndef UINTMAX_C // [ +# define UINTMAX_C UINT64_C +#endif // UINTMAX_C ] + +#endif // __STDC_CONSTANT_MACROS ] + +#endif // _MSC_VER >= 1600 ] + +#endif // _MSC_STDINT_H_ ] diff -Nru c-blosc-1.17.1+ds1/build.py c-blosc-1.21.1+ds2/build.py --- c-blosc-1.17.1+ds1/build.py 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/build.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,10 +0,0 @@ -from conan.packager import ConanMultiPackager -import os - -if __name__ == "__main__": - version = os.getenv("TRAVIS_TAG") or os.getenv("APPVEYOR_REPO_TAG_NAME") or "dev" - reference = "c-blosc/%s" % version - upload = os.getenv("CONAN_UPLOAD") if (version != "dev") else False - builder = ConanMultiPackager(reference=reference, upload=upload) - builder.add_common_builds(shared_option_name="c-blosc:shared") - builder.run() diff -Nru c-blosc-1.17.1+ds1/CMakeLists.txt c-blosc-1.21.1+ds2/CMakeLists.txt --- c-blosc-1.17.1+ds1/CMakeLists.txt 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/CMakeLists.txt 2021-10-06 08:16:29.000000000 +0000 @@ -9,6 +9,8 @@ # build the shared library version of the Blosc library # BUILD_TESTS: default ON # build test programs and generates the "test" target +# BUILD_FUZZERS: default ON +# build fuzz test programs and generates the "test" target # BUILD_BENCHMARKS: default ON # build the benchmark program # DEACTIVATE_SSE2: default OFF @@ -17,7 +19,7 @@ # do not attempt to build with AVX2 instructions # DEACTIVATE_LZ4: default OFF # do not include support for the LZ4 library -# DEACTIVATE_SNAPPY: default OFF +# DEACTIVATE_SNAPPY: default ON # do not include support for the Snappy library # DEACTIVATE_ZLIB: default OFF # do not include support for the Zlib library @@ -28,9 +30,6 @@ # PREFER_EXTERNAL_LZ4: default OFF # when found, use the installed LZ4 libs instead of included # sources -# PREFER_EXTERNAL_SNAPPY: default OFF -# when found, use the installed Snappy libs instead of included -# sources # PREFER_EXTERNAL_ZLIB: default OFF # when found, use the installed zlib libs instead of included # sources @@ -75,7 +74,7 @@ if (NOT CMAKE_VERSION VERSION_LESS 3.3) cmake_policy(SET CMP0063 NEW) endif() -project(blosc) +project(blosc C) # parse the full version numbers from blosc.h file(READ ${CMAKE_CURRENT_SOURCE_DIR}/blosc/blosc.h _blosc_h_contents) @@ -96,27 +95,27 @@ option(BUILD_SHARED "Build a shared library version of the blosc library." ON) option(BUILD_TESTS - "Build test programs form the blosc compression library" ON) + "Build test programs from the blosc compression library" ON) +option(BUILD_FUZZERS + "Build fuzzer programs from the blosc compression library" ${BUILD_STATIC}) option(BUILD_BENCHMARKS - "Build benchmark programs form the blosc compression library" ON) + "Build benchmark programs from the blosc compression library" ON) option(DEACTIVATE_SSE2 - "Do not attempt to build with SSE2 instructions" OFF) + "Do not attempt to build with SSE2 instructions" OFF) option(DEACTIVATE_AVX2 - "Do not attempt to build with AVX2 instructions" OFF) + "Do not attempt to build with AVX2 instructions" OFF) option(DEACTIVATE_LZ4 "Do not include support for the LZ4 library." OFF) option(DEACTIVATE_SNAPPY - "Do not include support for the Snappy library." OFF) + "Do not include support for the Snappy library." ON) option(DEACTIVATE_ZLIB "Do not include support for the Zlib library." OFF) option(DEACTIVATE_ZSTD - "Do not include support for the Zstd library." OFF) + "Do not include support for the Zstd library." OFF) option(DEACTIVATE_SYMBOLS_CHECK - "Do not check for symbols in shared or static libraries." ON) + "Do not check for symbols in shared or static libraries." ON) option(PREFER_EXTERNAL_LZ4 "Find and use external LZ4 library instead of included sources." OFF) -option(PREFER_EXTERNAL_SNAPPY - "Find and use external Snappy library instead of included sources." OFF) option(PREFER_EXTERNAL_ZLIB "Find and use external Zlib library instead of included sources." OFF) option(PREFER_EXTERNAL_ZSTD @@ -137,14 +136,13 @@ endif(NOT DEACTIVATE_LZ4) if(NOT DEACTIVATE_SNAPPY) - if(PREFER_EXTERNAL_SNAPPY) - find_package(Snappy) + find_package(Snappy) + if(SNAPPY_FOUND) + message(STATUS "Activating support for SNAPPY.") + set(HAVE_SNAPPY TRUE) else() - message(STATUS "Using Snappy internal sources.") - endif(PREFER_EXTERNAL_SNAPPY) - # HAVE_SNAPPY will be set to true because even if the library is not found, - # we will use the included sources for it - set(HAVE_SNAPPY TRUE) + message(STATUS "SNAPPY *not* found. De-activating support for it.") + endif() endif(NOT DEACTIVATE_SNAPPY) if(NOT DEACTIVATE_ZLIB) @@ -255,8 +253,8 @@ set(COMPILER_SUPPORT_SSE2 FALSE) endif() -# disable AVX2 if specified -if(DEACTIVATE_AVX2) +# disable AVX2 if specified or if SSE is deactivated +if(DEACTIVATE_AVX2 OR DEACTIVATE_SSE2) set(COMPILER_SUPPORT_AVX2 FALSE) endif() @@ -324,6 +322,14 @@ add_subdirectory(compat) endif(BUILD_TESTS) +if(BUILD_FUZZERS) + if(NOT BUILD_STATIC) + message(FATAL_ERROR "BUILD_FUZZERS requires BUILD_STATIC to be enabled.") + endif() + enable_testing() + add_subdirectory(tests/fuzz) +endif(BUILD_FUZZERS) + if(BUILD_BENCHMARKS) add_subdirectory(bench) endif(BUILD_BENCHMARKS) diff -Nru c-blosc-1.17.1+ds1/code_of_conduct.md c-blosc-1.21.1+ds2/code_of_conduct.md --- c-blosc-1.17.1+ds1/code_of_conduct.md 1970-01-01 00:00:00.000000000 +0000 +++ c-blosc-1.21.1+ds2/code_of_conduct.md 2021-10-06 08:16:29.000000000 +0000 @@ -0,0 +1,5 @@ +# Code of Conduct + +The Blosc community has adopted a Code of Conduct that we expect project participants to adhere to. +Please read the [full text](https://github.com/Blosc/community/blob/master/code_of_conduct.md) +so that you can understand what actions will and will not be tolerated. diff -Nru c-blosc-1.17.1+ds1/CODE_OF_CONDUCT.md c-blosc-1.21.1+ds2/CODE_OF_CONDUCT.md --- c-blosc-1.17.1+ds1/CODE_OF_CONDUCT.md 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/CODE_OF_CONDUCT.md 1970-01-01 00:00:00.000000000 +0000 @@ -1,5 +0,0 @@ -# Code of Conduct - -The Blosc community has adopted a Code of Conduct that we expect project participants to adhere to. -Please read the [full text](https://github.com/Blosc/CodeOfConduct/README.md) -so that you can understand what actions will and will not be tolerated. Binary files /tmp/tmpmy3y61tj/lkzE1Reo8j/c-blosc-1.17.1+ds1/compat/blosc-1.18.0-blosclz-bitshuffle.cdata and /tmp/tmpmy3y61tj/DQn15A5Dqo/c-blosc-1.21.1+ds2/compat/blosc-1.18.0-blosclz-bitshuffle.cdata differ Binary files /tmp/tmpmy3y61tj/lkzE1Reo8j/c-blosc-1.17.1+ds1/compat/blosc-1.18.0-blosclz.cdata and /tmp/tmpmy3y61tj/DQn15A5Dqo/c-blosc-1.21.1+ds2/compat/blosc-1.18.0-blosclz.cdata differ Binary files /tmp/tmpmy3y61tj/lkzE1Reo8j/c-blosc-1.17.1+ds1/compat/blosc-1.18.0-lz4-bitshuffle.cdata and /tmp/tmpmy3y61tj/DQn15A5Dqo/c-blosc-1.21.1+ds2/compat/blosc-1.18.0-lz4-bitshuffle.cdata differ Binary files /tmp/tmpmy3y61tj/lkzE1Reo8j/c-blosc-1.17.1+ds1/compat/blosc-1.18.0-lz4.cdata and /tmp/tmpmy3y61tj/DQn15A5Dqo/c-blosc-1.21.1+ds2/compat/blosc-1.18.0-lz4.cdata differ Binary files /tmp/tmpmy3y61tj/lkzE1Reo8j/c-blosc-1.17.1+ds1/compat/blosc-1.18.0-lz4hc.cdata and /tmp/tmpmy3y61tj/DQn15A5Dqo/c-blosc-1.21.1+ds2/compat/blosc-1.18.0-lz4hc.cdata differ Binary files /tmp/tmpmy3y61tj/lkzE1Reo8j/c-blosc-1.17.1+ds1/compat/blosc-1.18.0-zlib.cdata and /tmp/tmpmy3y61tj/DQn15A5Dqo/c-blosc-1.21.1+ds2/compat/blosc-1.18.0-zlib.cdata differ Binary files /tmp/tmpmy3y61tj/lkzE1Reo8j/c-blosc-1.17.1+ds1/compat/blosc-1.18.0-zstd.cdata and /tmp/tmpmy3y61tj/DQn15A5Dqo/c-blosc-1.21.1+ds2/compat/blosc-1.18.0-zstd.cdata differ diff -Nru c-blosc-1.17.1+ds1/compat/CMakeLists.txt c-blosc-1.21.1+ds2/compat/CMakeLists.txt --- c-blosc-1.17.1+ds1/compat/CMakeLists.txt 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/compat/CMakeLists.txt 2021-10-06 08:16:29.000000000 +0000 @@ -20,6 +20,13 @@ if (TEST_INCLUDE_COMPAT) file(GLOB DATAFILES *.cdata) foreach(datafile ${DATAFILES}) + # Don't test data if compressor is deactivated + if((datafile MATCHES "lz4" AND DEACTIVATE_LZ4) OR + (datafile MATCHES "snappy" AND DEACTIVATE_SNAPPY) OR + (datafile MATCHES "zlib" AND DEACTIVATE_ZLIB) OR + (datafile MATCHES "zstd" AND DEACTIVATE_ZSTD)) + continue() + endif() get_filename_component(fname ${datafile} NAME) add_test(test_compat_${fname} filegen decompress ${datafile}) endforeach(datafile) diff -Nru c-blosc-1.17.1+ds1/COMPILING_WITH_WHEELS.rst c-blosc-1.21.1+ds2/COMPILING_WITH_WHEELS.rst --- c-blosc-1.17.1+ds1/COMPILING_WITH_WHEELS.rst 1970-01-01 00:00:00.000000000 +0000 +++ c-blosc-1.21.1+ds2/COMPILING_WITH_WHEELS.rst 2021-10-06 08:16:29.000000000 +0000 @@ -0,0 +1,233 @@ +C-Blosc libraries come with Python-Blosc wheels +=============================================== + +Starting on version 1.21.0, C-Blosc binary libraries can easily be installed from Python-Blosc (>= 1.10) wheels: + +.. code-block:: console + + $ pip install blosc (base) + Collecting blosc + Downloading blosc-1.10.0-cp37-cp37m-macosx_10_9_x86_64.whl (2.2 MB) + |████████████████████████████████| 2.2 MB 4.7 MB/s + Installing collected packages: blosc + Attempting uninstall: blosc + Found existing installation: blosc 1.10.0 + Uninstalling blosc-1.10.0: + Successfully uninstalled blosc-1.10.0 + Successfully installed blosc-1.10.0 + +As a result, one can easily update to the latest version of C-Blosc binaries without the need to manually compile the thing. Following are instructions on how to use the libraries in wheels for different platforms. + + +Compiling C files with Blosc wheels on Windows +---------------------------------------------- + +- The wheels for Windows have been produced with the Microsoft MSVC compiler, so we recommend that you use it too. You can get it for free at: https://visualstudio.microsoft.com/es/downloads/. + +- In order to check that the MSVC command line is set up correctly, enter ``cl`` in the command prompt window and verify that the output looks something like this: + +.. code-block:: console + + > cl + Microsoft (R) C/C++ Optimizing Compiler Version 19.00.24245 for x64 + Copyright (C) Microsoft Corporation. All rights reserved. + + usage: cl [ option... ] filename... [ /link linkoption... ] + +- Now, install the wheels: + +.. code-block:: console + + > pip install blosc + Collecting blosc + Using cached blosc-1.10.0-cp37-cp37m-win_amd64.whl (1.5 MB) + Installing collected packages: blosc + Successfully installed blosc-1.10.0 + +- Make the compiler available. Its typical installation location uses to be `C:\\Program files (x86)\\Microsoft Visual Studio`, so change your current directory there. Then, to set up the build architecture environment you can open a command prompt window in the `VC\\Auxiliary\\Build` subdirectory and execute `vcvarsall.bat x64` if your achitecture is 64 bits or `vcvarsall.bat x86` if it is 32 bits. + +- You will need to know the path where the Blosc wheel has installed its files. For this we will use the `dir /s` command (but you can use your preferred location method): + +.. code-block:: console + + > dir /s c:\blosc.lib + Volume in drive C is OS + Volume Serial Number is 7A21-A5D5 + + Directory of c:\Users\user\miniconda3\Lib + + 14/12/2020 09:56 7.022 blosc.lib + 1 File(s) 7.022 bytes + + Total list files: + 1 File(s) 7.022 bytes + 0 dirs 38.981.902.336 free bytes + +- The output shows the path of blosc.lib in your system, but we are rather interested in the parent one: + +.. code-block:: console + + > set WHEEL_DIR=c:\Users\user\miniconda3 + +- Now, it is important to copy the library `blosc.dll` to C:\\Windows\\System32 directory, so it can be found by the executable when it is necessary. + +- Finally, to compile C files using Blosc libraries, enter this command: + +.. code-block:: console + + > cl .c /Ox /Fe.exe /I /MT /link/NODEFAULTLIB:MSVCRT + +- For instance, in the case of blosc "examples/simple.c": + +.. code-block:: console + + > cl simple.c %WHEEL_DIR%\lib\blosc.lib /Ox /Fesimple.exe /I%WHEEL_DIR%\include /MT /link/NODEFAULTLIB:MSVCRT + + Microsoft (R) C/C++ Optimizing Compiler Version 19.10.25017 for x86 + Copyright (C) Microsoft Corporation. All rights reserved. + + simple.c + Microsoft (R) Incremental Linker Version 14.10.25017.0 + Copyright (C) Microsoft Corporation. All rights reserved. + + /out:simple.exe + simple.obj + /NODEFAULTLIB:MSVCRT + .\miniconda3\lib\blosc.lib + +- And you can run your program: + +.. code-block:: console + + > simple + Blosc version info: 1.20.1 ($Date:: 2020-09-08 #$) + Compression: 4000000 -> 37816 (105.8x) + Decompression succesful! + Succesful roundtrip! + +- Rejoice! + + +Compiling C files with Blosc wheels on Linux +-------------------------------------------- + +- Install the wheels: + +.. code-block:: console + + $ pip install blosc + Collecting blosc + Using cached blosc-1.10.0-cp37-cp37m-manylinux2010_x86_64.whl (2.2 MB) + Installing collected packages: blosc + Successfully installed blosc-1.10.0 + +- Find the path where blosc wheel has installed its files: + +.. code-block:: console + + $ find / -name libblosc.so 2>/dev/null + /home/soscar/miniconda3/lib/libblosc.so + +- The output shows the path of libblosc.so, but we are rather interested in the parent one: + +.. code-block:: console + + $ WHEEL_DIR=/home/soscar/miniconda3 + +- To compile C files using blosc you only need to enter the commands: + +.. code-block:: console + + $ export LD_LIBRARY_PATH= + $ gcc .c -I -o -L -lblosc + +- For instance, let's compile blosc's "examples/many_compressors.c": + +.. code-block:: console + + $ export LD_LIBRARY_PATH=$WHEEL_DIR/lib # note that you need the LD_LIBRARY_PATH env variable + $ gcc many_compressors.c -I$WHEEL_DIR/include -o many_compressors -L$WHEEL_DIR/lib -lblosc + +- Run your program: + +.. code-block:: console + + $ ./many_compressors + Blosc version info: 1.20.1 ($Date:: 2020-09-08 #$) + Using 4 threads (previously using 1) + Using blosclz compressor + Compression: 4000000 -> 37816 (105.8x) + Succesful roundtrip! + Using lz4 compressor + Compression: 4000000 -> 37938 (105.4x) + Succesful roundtrip! + Using lz4hc compressor + Compression: 4000000 -> 27165 (147.2x) + Succesful roundtrip! + +- Rejoice! + + +Compiling C files with Blosc wheels on MacOS +-------------------------------------------- + +- Install the wheels: + +.. code-block:: console + + $ pip install blosc (base) + Collecting blosc + Downloading blosc-1.10.0-cp37-cp37m-macosx_10_9_x86_64.whl (2.2 MB) + |████████████████████████████████| 2.2 MB 4.7 MB/s + Installing collected packages: blosc + Attempting uninstall: blosc + Found existing installation: blosc 1.10.0 + Uninstalling blosc-1.10.0: + Successfully uninstalled blosc-1.10.0 + Successfully installed blosc-1.10.0 + +- Find the path where blosc wheel has installed its files: + +.. code-block:: console + + $ find / -name libblosc.dylib 2>/dev/null + /home/soscar/miniconda3/lib/libblosc.dylib + +- The output shows the path of libblosc.dylib, but we are rather interested in the parent one: + +.. code-block:: console + + $ WHEEL_DIR=/home/soscar/miniconda3 + +- To compile C files using blosc you only need to enter the commands: + +.. code-block:: console + + $ export LD_LIBRARY_PATH= + $ clang .c -I -o -L -lblosc + +- For instance, let's compile blosc's "examples/many_compressors.c": + +.. code-block:: console + + $ export LD_LIBRARY_PATH=$WHEEL_DIR/lib # note that you need the LD_LIBRARY_PATH env variable + $ clang many_compressors.c -I$WHEEL_DIR/include -o many_compressors -L$WHEEL_DIR/lib -lblosc + +- Run your program: + +.. code-block:: console + + $ ./many_compressors + Blosc version info: 1.20.1 ($Date:: 2020-09-08 #$) + Using 4 threads (previously using 1) + Using blosclz compressor + Compression: 4000000 -> 37816 (105.8x) + Succesful roundtrip! + Using lz4 compressor + Compression: 4000000 -> 37938 (105.4x) + Succesful roundtrip! + Using lz4hc compressor + Compression: 4000000 -> 27165 (147.2x) + Succesful roundtrip! + +- Rejoice! diff -Nru c-blosc-1.17.1+ds1/conanfile.py c-blosc-1.21.1+ds2/conanfile.py --- c-blosc-1.17.1+ds1/conanfile.py 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/conanfile.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,74 +0,0 @@ -import os -from conans import ConanFile, CMake, tools - - -class CbloscConan(ConanFile): - name = "c-blosc" - description = "An extremely fast, multi-threaded, meta-compressor library" - license = "BSD" - url = "https://github.com/Blosc/c-blosc" - settings = "os", "compiler", "build_type", "arch" - options = {"shared": [True, False]} - default_options = "shared=False" - generators = "cmake" - exports_sources = "*", "!test_package/*", "!appveyor*", "!.*.yml", "!bench/plot-speeds.py", "!build.py", "!.*" - - @property - def run_tests(self): - return "CONAN_RUN_TESTS" in os.environ - - def build(self): - os.mkdir("build") - tools.replace_in_file("CMakeLists.txt", "project(blosc)", '''project(blosc) - include(${CMAKE_BINARY_DIR}/../conanbuildinfo.cmake) - conan_basic_setup(NO_OUTPUT_DIRS)''') - cmake = CMake(self) - cmake.definitions["BUILD_TESTS"] = "ON" if self.run_tests else "OFF" - cmake.definitions["BUILD_BENCHMARKS"] = "ON" if self.run_tests else "OFF" - cmake.definitions["BUILD_SHARED"] = "ON" if (self.options.shared or self.run_tests) else "OFF" - cmake.definitions["BUILD_STATIC"] = "OFF" if self.options.shared else "ON" - cmake.configure(build_folder="build") - cmake.build() - - if self.run_tests: - self.output.warn("Running tests!!") - self.launch_tests() - - def launch_tests(self): - """Conan will remove rpaths from shared libs to be able to reuse the shared libs, we need - to tell the tests where to find the shared libs""" - test_args = "-VV" if tools.os_info.is_windows else "" - with tools.chdir("build"): - outdir = os.path.join(self.build_folder, "build", "blosc") - if tools.os_info.is_macos: - prefix = "DYLD_LIBRARY_PATH=%s" % outdir - elif tools.os_info.is_windows: - prefix = "PATH=%s;%%PATH%%" % outdir - elif tools.os_info.is_linux: - prefix = "LD_LIBRARY_PATH=%s" % outdir - else: - return - with tools.environment_append({'CTEST_OUTPUT_ON_FAILURE': '1'}): - self.run("%s ctest %s" % (prefix, test_args)) - - def package(self): - self.copy("blosc.h", dst="include", src="blosc") - self.copy("blosc-export.h", dst="include", src="blosc") - self.copy("*libblosc.a", dst="lib", keep_path=False) - - if self.options.shared: - self.copy("*/blosc.lib", dst="lib", keep_path=False) - self.copy("*blosc.dll", dst="bin", keep_path=False) - self.copy("*blosc.*dylib*", dst="lib", keep_path=False, symlinks=True) - self.copy("*blosc.so*", dst="lib", keep_path=False, symlinks=True) - self.copy("*libblosc.dll.a", dst="lib", keep_path=False) # Mingw - else: - self.copy("*libblosc.lib", dst="lib", src="", keep_path=False) - - def package_info(self): - if self.settings.compiler == "Visual Studio" and not self.options.shared: - self.cpp_info.libs = ["libblosc"] - else: - self.cpp_info.libs = ["blosc"] - if self.settings.os == "Linux": - self.cpp_info.libs.append("pthread") diff -Nru c-blosc-1.17.1+ds1/debian/changelog c-blosc-1.21.1+ds2/debian/changelog --- c-blosc-1.17.1+ds1/debian/changelog 2020-07-31 06:43:05.000000000 +0000 +++ c-blosc-1.21.1+ds2/debian/changelog 2022-09-21 19:50:07.000000000 +0000 @@ -1,9 +1,80 @@ -c-blosc (1.17.1+ds1-1~18.04.sav0) bionic; urgency=medium +c-blosc (1.21.1+ds2-2~18.04.sav0) bionic; urgency=medium * Backport to Bionic * debian/control: Set debhelper-compat (= 11) BD - -- Rob Savoury Thu, 30 Jul 2020 23:43:05 -0700 + -- Rob Savoury Wed, 21 Sep 2022 12:50:07 -0700 + +c-blosc (1.21.1+ds2-2) unstable; urgency=medium + + * Compile with '-O1' on mips64el. Closes: #1000953 + + -- Håvard Flaget Aasen Mon, 06 Dec 2021 14:58:56 +0100 + +c-blosc (1.21.1+ds2-1) unstable; urgency=medium + + [ Håvard Flaget Aasen ] + * Change repacking of source. Only remove third-party libraries. + * Add pkgconfig file. + * d/copyright: + - Include files from blosc/win32/*, blosc/fastcopy.c and blosc/blosclz.* + - Be more specific regarding blosc/bitshuffle* files. + * d/rules: + - Verbose build, default to off. + - Remove commented out configure options. + + [ Mathieu Malaterre ] + * d/patches: blosc should support multiarch. Closes: #836575 + * d/symbols: Add missing symbol file + + + -- Håvard Flaget Aasen Fri, 26 Nov 2021 15:20:38 +0100 + +c-blosc (1.21.1+ds1-1) unstable; urgency=medium + + * New upstream release + * Adopt package. Closes: #945176 + * d/watch: Update URL to GitHub. + * d/control: + - Drop version constriction on build-dependency + - Update Standards-Version to 4.6.0 + * d/copyright: Update mail address. + + -- Håvard Flaget Aasen Wed, 06 Oct 2021 21:22:31 +0200 + +c-blosc (1.20.1+ds1-2) unstable; urgency=medium + + * d/rules: Change CMake flag to include support for snappy Closes: #976569 + + -- Håvard Flaget Aasen Sun, 06 Dec 2020 17:43:44 +0100 + +c-blosc (1.20.1+ds1-1) unstable; urgency=medium + + * QA upload. + [ Debian Janitor ] + * Use secure URI in Homepage field. + * debian/copyright: use spaces rather than tabs to start continuation + lines. + * Set upstream metadata fields: Bug-Database. + * Set upstream metadata fields: Bug-Submit. + * Update standards version to 4.5.0, no changes needed. + + [ Håvard Flaget Aasen ] + * New upstream version 1.20.1+ds1 + * d/control: + - Bump debhelper to 13 + - Add rules-Requires-Root: no + - Update Standards-Version to 4.5.1 + * d/watch: Update to version 4 + * Set upstream metadata fields: Repository, Repository-Browse + * d/rules, d/libblosc-dev.docs: Rename documentation files, to match + changes upstream. + * d/not-installed: New file, explicitly mention files which is not installed. + * d/copyright: + - Add Upstream-Contact. + - Add Blosc Development Team in Copyright field + + -- Håvard Flaget Aasen Fri, 27 Nov 2020 16:52:53 +0100 c-blosc (1.17.1+ds1-1) unstable; urgency=medium diff -Nru c-blosc-1.17.1+ds1/debian/control c-blosc-1.21.1+ds2/debian/control --- c-blosc-1.17.1+ds1/debian/control 2020-07-31 06:43:05.000000000 +0000 +++ c-blosc-1.21.1+ds2/debian/control 2022-09-21 19:50:07.000000000 +0000 @@ -1,18 +1,19 @@ Source: c-blosc Priority: optional Section: libs -Maintainer: Debian QA Group +Maintainer: Håvard Flaget Aasen Build-Depends: debhelper-compat (= 11), cmake, - liblz4-dev (>= 0.0~r130), + liblz4-dev, libsnappy-dev, zlib1g-dev, libzstd-dev, python3-docutils , links , pandoc -Standards-Version: 4.4.1 -Homepage: http://blosc.org/ +Standards-Version: 4.6.0 +Rules-Requires-Root: no +Homepage: https://blosc.org/ Vcs-Browser: https://salsa.debian.org/debian/c-blosc Vcs-Git: https://salsa.debian.org/debian/c-blosc.git @@ -39,6 +40,7 @@ Blosc. Package: libblosc1 +Multi-Arch: same Architecture: any Depends: ${shlibs:Depends}, ${misc:Depends} diff -Nru c-blosc-1.17.1+ds1/debian/copyright c-blosc-1.21.1+ds2/debian/copyright --- c-blosc-1.17.1+ds1/debian/copyright 2019-12-12 21:04:59.000000000 +0000 +++ c-blosc-1.21.1+ds2/debian/copyright 2021-12-06 13:58:56.000000000 +0000 @@ -1,13 +1,39 @@ Format: https://www.debian.org/doc/packaging-manuals/copyright-format/1.0/ Upstream-Name: c-blosc +Upstream-Contact: Blosc Development Team Source: https://github.com/Blosc/c-blosc -Files-Excluded: appveyor - appveyor.yml - blosc/win32 - internal-complibs +Files-Excluded: internal-complibs Files: * -Copyright: 2014-2018 Francesc Alted +Copyright: 2014-2018 Francesc Alted + 2019-present Blosc Development Team +License: BSD-3-clause + +Files: blosc/bitshuffle-avx2.c + blosc/bitshuffle-sse2.c +Copyright: 2014 Kiyoshi Masui +License: Expat +Comment: Adapted for c-blosc by Francesc Alted. + +Files: blosc/blosclz.* +Copyright: 2005-2007 Ariya Hidayat (ariya@kde.org) + 2014-2018 Francesc Alted + 2019-present Blosc Development Team +License: Expat and BSD-3-clause +Comment: Code is based on FastLZ + +Files: blosc/fastcopy.c +Copyright: 1995-2013 Jean-loup Gailly and Mark Adler + 2018 Francesc Alted +License: Zlib and BSD-3-clause +Comment: Based on memcopy.h from the zlib-ng compression library. + +Files: blosc/win32/pthread.* +Copyright: 2009 Andrzej K. Haczewski +License: Expat + +Files: blosc/win32/stdint-windows.h +Copyright: 2006-2013 Alexander Chemeris License: BSD-3-clause Files: debian/* @@ -41,18 +67,13 @@ (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. -Files: blosc/bitshuffle-* -Copyright: 2014 Kiyoshi Masui - 2014 Francesc Alted -License: Expat - License: Expat - Permission is hereby granted, free of charge, to any person - obtaining a copy of this software and associated documentation files - (the "Software"), to deal in the Software without restriction, including + Permission is hereby granted, free of charge, to any person + obtaining a copy of this software and associated documentation files + (the "Software"), to deal in the Software without restriction, including without limitation the rights to use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies of the Software, and to permit - persons to whom the Software is furnished to do so, subject to the + persons to whom the Software is furnished to do so, subject to the following conditions: . The above copyright notice and this permission notice shall be included in @@ -66,3 +87,19 @@ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +License: Zlib + This software is provided 'as-is', without any express or implied + warranty. In no event will the authors be held liable for any damages + arising from the use of this software. + . + Permission is granted to anyone to use this software for any purpose, + including commercial applications, and to alter it and redistribute it + freely, subject to the following restrictions: + . + 1. The origin of this software must not be misrepresented; you must not + claim that you wrote the original software. If you use this software + in a product, an acknowledgment in the product documentation would be + appreciated but is not required. + 2. Altered source versions must be plainly marked as such, and must not be + misrepresented as being the original software. + 3. This notice may not be removed or altered from any source distribution. diff -Nru c-blosc-1.17.1+ds1/debian/libblosc1.install c-blosc-1.21.1+ds2/debian/libblosc1.install --- c-blosc-1.17.1+ds1/debian/libblosc1.install 2019-12-12 21:04:59.000000000 +0000 +++ c-blosc-1.21.1+ds2/debian/libblosc1.install 2021-12-06 13:58:56.000000000 +0000 @@ -1 +1 @@ -usr/lib/lib*.so.* +usr/lib/*/lib*.so.* diff -Nru c-blosc-1.17.1+ds1/debian/libblosc1.symbols c-blosc-1.21.1+ds2/debian/libblosc1.symbols --- c-blosc-1.17.1+ds1/debian/libblosc1.symbols 1970-01-01 00:00:00.000000000 +0000 +++ c-blosc-1.21.1+ds2/debian/libblosc1.symbols 2021-12-06 13:58:56.000000000 +0000 @@ -0,0 +1,27 @@ +libblosc.so.1 libblosc1 #MINVER# +* Build-Depends-Package: libblosc-dev + blosc_cbuffer_complib@Base 1.7.0 + blosc_cbuffer_metainfo@Base 1.7.0 + blosc_cbuffer_sizes@Base 1.7.0 + blosc_cbuffer_validate@Base 1.17.1+ds1 + blosc_cbuffer_versions@Base 1.7.0 + blosc_compcode_to_compname@Base 1.7.0 + blosc_compname_to_compcode@Base 1.7.0 + blosc_compress@Base 1.7.0 + blosc_compress_ctx@Base 1.7.0 + blosc_decompress@Base 1.7.0 + blosc_decompress_ctx@Base 1.7.0 + blosc_destroy@Base 1.7.0 + blosc_free_resources@Base 1.7.0 + blosc_get_blocksize@Base 1.9.0 + blosc_get_complib_info@Base 1.7.0 + blosc_get_compressor@Base 1.9.0 + blosc_get_nthreads@Base 1.9.0 + blosc_get_version_string@Base 1.7.0 + blosc_getitem@Base 1.7.0 + blosc_init@Base 1.7.0 + blosc_list_compressors@Base 1.7.0 + blosc_set_blocksize@Base 1.7.0 + blosc_set_compressor@Base 1.7.0 + blosc_set_nthreads@Base 1.7.0 + blosc_set_splitmode@Base 1.14.0+ds1 diff -Nru c-blosc-1.17.1+ds1/debian/libblosc-dev.docs c-blosc-1.21.1+ds2/debian/libblosc-dev.docs --- c-blosc-1.17.1+ds1/debian/libblosc-dev.docs 2019-12-12 21:04:59.000000000 +0000 +++ c-blosc-1.21.1+ds2/debian/libblosc-dev.docs 2021-12-06 13:58:56.000000000 +0000 @@ -1,2 +1,2 @@ -debian/tmp/README_HEADER -debian/tmp/README_THREADED \ No newline at end of file +debian/tmp/README_CHUNK_FORMAT +debian/tmp/README_THREADED diff -Nru c-blosc-1.17.1+ds1/debian/libblosc-dev.install c-blosc-1.21.1+ds2/debian/libblosc-dev.install --- c-blosc-1.17.1+ds1/debian/libblosc-dev.install 2019-12-12 21:04:59.000000000 +0000 +++ c-blosc-1.21.1+ds2/debian/libblosc-dev.install 2021-12-06 13:58:56.000000000 +0000 @@ -1,5 +1,4 @@ usr/include/ -usr/lib/lib*.a -usr/lib/lib*.so -#usr/lib/pkgconfig/* -#usr/share/pkgconfig/* +usr/lib/*/lib*.a +usr/lib/*/lib*.so +usr/lib/*/pkgconfig/* diff -Nru c-blosc-1.17.1+ds1/debian/not-installed c-blosc-1.21.1+ds2/debian/not-installed --- c-blosc-1.17.1+ds1/debian/not-installed 1970-01-01 00:00:00.000000000 +0000 +++ c-blosc-1.21.1+ds2/debian/not-installed 2021-12-06 13:58:56.000000000 +0000 @@ -0,0 +1,3 @@ +README_THREADED.html +README_CHUNK_FORMAT.html +THANKS.html diff -Nru c-blosc-1.17.1+ds1/debian/patches/multiarch.patch c-blosc-1.21.1+ds2/debian/patches/multiarch.patch --- c-blosc-1.17.1+ds1/debian/patches/multiarch.patch 1970-01-01 00:00:00.000000000 +0000 +++ c-blosc-1.21.1+ds2/debian/patches/multiarch.patch 2021-12-06 13:58:56.000000000 +0000 @@ -0,0 +1,59 @@ +From: Mathieu Malaterre +Date: Tue, 23 Nov 2021 20:49:52 +0100 +Subject: blosc should support multiarch + +Bug-Debian: https://bugs.debian.org/836575 +Forwarded: no +Last-Update: 2021-11-23 +--- + CMakeLists.txt | 3 ++- + blosc.pc.in | 2 +- + blosc/CMakeLists.txt | 2 +- + 3 files changed, 4 insertions(+), 3 deletions(-) + +diff --git a/CMakeLists.txt b/CMakeLists.txt +index f5eff9e..178eb7a 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -75,6 +75,7 @@ if (NOT CMAKE_VERSION VERSION_LESS 3.3) + cmake_policy(SET CMP0063 NEW) + endif() + project(blosc C) ++include(GNUInstallDirs) + + # parse the full version numbers from blosc.h + file(READ ${CMAKE_CURRENT_SOURCE_DIR}/blosc/blosc.h _blosc_h_contents) +@@ -342,7 +343,7 @@ if (BLOSC_INSTALL) + "${CMAKE_CURRENT_BINARY_DIR}/blosc.pc" + @ONLY) + install(FILES "${CMAKE_CURRENT_BINARY_DIR}/blosc.pc" +- DESTINATION lib/pkgconfig COMPONENT DEV) ++ DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig" COMPONENT DEV) + + configure_file( + "${CMAKE_CURRENT_SOURCE_DIR}/cmake_uninstall.cmake.in" +diff --git a/blosc.pc.in b/blosc.pc.in +index c01273b..8570df1 100644 +--- a/blosc.pc.in ++++ b/blosc.pc.in +@@ -1,6 +1,6 @@ + prefix=@CMAKE_INSTALL_PREFIX@ + exec_prefix=${prefix} +-libdir=${exec_prefix}/lib ++libdir=${exec_prefix}/@CMAKE_INSTALL_LIBDIR@ + sharedlibdir=${libdir} + includedir=${prefix}/include + +diff --git a/blosc/CMakeLists.txt b/blosc/CMakeLists.txt +index 84b72e5..d23b1be 100644 +--- a/blosc/CMakeLists.txt ++++ b/blosc/CMakeLists.txt +@@ -62,7 +62,7 @@ endif(COMPILER_SUPPORT_AVX2) + set(SOURCES ${SOURCES} shuffle.c) + + # library install directory +-set(lib_dir lib${LIB_SUFFIX}) ++set(lib_dir ${CMAKE_INSTALL_LIBDIR}) + set(version_string ${BLOSC_VERSION_MAJOR}.${BLOSC_VERSION_MINOR}.${BLOSC_VERSION_PATCH}) + + set(CMAKE_THREAD_PREFER_PTHREAD TRUE) # pre 3.1 diff -Nru c-blosc-1.17.1+ds1/debian/patches/series c-blosc-1.21.1+ds2/debian/patches/series --- c-blosc-1.17.1+ds1/debian/patches/series 1970-01-01 00:00:00.000000000 +0000 +++ c-blosc-1.21.1+ds2/debian/patches/series 2021-12-06 13:58:56.000000000 +0000 @@ -0,0 +1 @@ +multiarch.patch diff -Nru c-blosc-1.17.1+ds1/debian/rules c-blosc-1.21.1+ds2/debian/rules --- c-blosc-1.17.1+ds1/debian/rules 2019-12-12 21:04:59.000000000 +0000 +++ c-blosc-1.21.1+ds2/debian/rules 2021-12-06 13:58:56.000000000 +0000 @@ -1,7 +1,18 @@ #!/usr/bin/make -f -export DH_VERBOSE = 1 + +#export DH_VERBOSE = 1 export DEB_BUILD_MAINT_OPTIONS = hardening=+all +DEB_HOST_ARCH ?= $(shell dpkg-architecture -qDEB_HOST_ARCH) + +# Testsuite started to fail on mips64el, possibly related to switch +# from GCC-10 to GCC-11. Tested that both -O1 and -O3 optimization +# worked. -O2 was successful only when disabling inline functions. +# See bug #1000953 +ifeq ($(DEB_HOST_ARCH),mips64el) + export DEB_CFLAGS_MAINT_APPEND = -O1 +endif + %: dh $@ --buildsystem=cmake @@ -10,14 +21,12 @@ -DCMAKE_BINARY_DIR=debian/tmp \ -DPREFER_EXTERNAL_LZ4=ON \ -DPREFER_EXTERNAL_ZSTD=ON \ - -DPREFER_EXTERNAL_SNAPPY=ON \ + -DDEACTIVATE_SNAPPY=OFF \ -DPREFER_EXTERNAL_ZLIB=ON -# -DCMAKE_LIBRARY_PATH=$(DEB_HOST_MULTIARCH) \ -# -DCMAKE_INSTALL_PREFIX=debian/libblosc1 \ override_dh_installdocs: ifeq (,$(filter nodoc, $(DEB_BUILD_OPTIONS))) - for i in README_HEADER README_THREADED THANKS; \ + for i in README_CHUNK_FORMAT README_THREADED THANKS; \ do rst2html -r 5 $$i.rst debian/tmp/$$i.html; \ LANG=C.UTF-8 links -dump debian/tmp/$$i.html > debian/tmp/$$i; done pandoc -f markdown -t plain -o debian/tmp/README README.md diff -Nru c-blosc-1.17.1+ds1/debian/upstream/metadata c-blosc-1.21.1+ds2/debian/upstream/metadata --- c-blosc-1.17.1+ds1/debian/upstream/metadata 1970-01-01 00:00:00.000000000 +0000 +++ c-blosc-1.21.1+ds2/debian/upstream/metadata 2021-12-06 13:58:56.000000000 +0000 @@ -0,0 +1,4 @@ +Bug-Database: https://github.com/Blosc/c-blosc/issues +Bug-Submit: https://github.com/Blosc/c-blosc/issues/new +Repository: https://github.com/Blosc/c-blosc.git +Repository-Browse: https://github.com/Blosc/c-blosc diff -Nru c-blosc-1.17.1+ds1/debian/watch c-blosc-1.21.1+ds2/debian/watch --- c-blosc-1.17.1+ds1/debian/watch 2019-12-12 21:04:59.000000000 +0000 +++ c-blosc-1.21.1+ds2/debian/watch 2021-12-06 13:58:56.000000000 +0000 @@ -1,5 +1,5 @@ -version=3 +version=4 opts=uversionmangle=s/(\d)[_\.\-\+]?((RC|rc|pre|dev|beta|alpha|b|a)[\-\.]?\d*)$/$1~$2/,\ -dversionmangle=s/\+ds1\d*$//,\ -repacksuffix=+ds1 \ -https://github.com/Blosc/c-blosc/tags .*/archive/v?(\d.*)\.(?:tgz|tbz2|txz|tar\.(?:gz|bz2|xz)) +dversionmangle=s/\+ds\d*$//,\ +repacksuffix=+ds2 \ +https://github.com/Blosc/@PACKAGE@/releases/latest .*/v?@ANY_VERSION@@ARCHIVE_EXT@ diff -Nru c-blosc-1.17.1+ds1/.github/workflows/cmake.yml c-blosc-1.21.1+ds2/.github/workflows/cmake.yml --- c-blosc-1.17.1+ds1/.github/workflows/cmake.yml 1970-01-01 00:00:00.000000000 +0000 +++ c-blosc-1.21.1+ds2/.github/workflows/cmake.yml 2021-10-06 08:16:29.000000000 +0000 @@ -0,0 +1,161 @@ +name: CI CMake +on: [push, pull_request] +jobs: + ci-cmake: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + strategy: + fail-fast: false + matrix: + name: [ + Ubuntu GCC, + Ubuntu GCC OSB, + Ubuntu GCC External LZ4, + Ubuntu GCC External SNAPPY, + Ubuntu GCC External ZLIB, + Ubuntu GCC External ZSTD, + Ubuntu Clang, + Ubuntu Clang No SSE2, + Ubuntu Clang No AVX2, + Ubuntu Clang No AVX2 No SSE2, + Ubuntu Clang No LZ4, + Ubuntu Clang No ZLIB, + Ubuntu Clang No ZSTD, + Windows MSVC Win32, + Windows MSVC Win64, + Windows GCC, + macOS Clang, + macOS GCC + ] + include: + - name: Ubuntu GCC + os: ubuntu-latest + compiler: gcc + + # Out of source build + - name: Ubuntu GCC OSB + os: ubuntu-latest + compiler: gcc + build-dir: ../build + build-src-dir: ../c-blosc + + - name: Ubuntu GCC External LZ4 + os: ubuntu-latest + compiler: gcc + packages: liblz4-1 liblz4-dev + cmake-args: -DPREFER_EXTERNAL_LZ4=ON + + - name: Ubuntu GCC External SNAPPY + os: ubuntu-latest + compiler: gcc + packages: libsnappy-dev + cmake-args: -DDEACTIVATE_SNAPPY=OFF + + - name: Ubuntu GCC External ZLIB + os: ubuntu-latest + compiler: gcc + packages: zlib1g-dev + cmake-args: -DPREFER_EXTERNAL_ZLIB=ON + + - name: Ubuntu GCC External ZSTD + os: ubuntu-latest + compiler: gcc + packages: zstd libzstd-dev + cmake-args: -DPREFER_EXTERNAL_ZSTD=ON + + - name: Ubuntu Clang + os: ubuntu-latest + compiler: clang + + - name: Ubuntu Clang No SSE2 + os: ubuntu-latest + compiler: clang + cmake-args: -DDEACTIVATE_SSE2=ON + + - name: Ubuntu Clang No AVX2 + os: ubuntu-latest + compiler: clang + cmake-args: -DDEACTIVATE_AVX2=ON + + - name: Ubuntu Clang No AVX2 No SSE2 + os: ubuntu-latest + compiler: clang + cmake-args: -DDEACTIVATE_AVX2=ON -DDEACTIVATE_SSE2=ON + + - name: Ubuntu Clang No LZ4 + os: ubuntu-latest + compiler: clang + cmake-args: -DDEACTIVATE_LZ4=ON + + - name: Ubuntu Clang No ZLIB + os: ubuntu-latest + compiler: clang + cmake-args: -DDEACTIVATE_ZLIB=ON + + - name: Ubuntu Clang No ZSTD + os: ubuntu-latest + compiler: clang + cmake-args: -DDEACTIVATE_ZSTD=ON + + - name: Windows MSVC Win32 + os: windows-latest + compiler: cl + cmake-args: -A Win32 + + - name: Windows MSVC Win64 + os: windows-latest + compiler: cl + cmake-args: -A x64 + + - name: Windows GCC + os: windows-latest + compiler: gcc + cmake-args: -G Ninja + + - name: macOS Clang + os: macOS-latest + compiler: clang + + - name: macOS GCC + os: macOS-latest + compiler: gcc + + steps: + - uses: actions/checkout@v1 + + - name: Install packages (Ubuntu) + if: runner.os == 'Linux' && matrix.packages + run: | + sudo apt-get update + sudo apt-get install -y ${{ matrix.packages }} + + - name: Install packages (Windows) + if: runner.os == 'Windows' + run: | + choco install ninja ${{ matrix.packages }} + + - name: Install packages (macOS) + if: runner.os == 'macOS' + run: | + brew install ninja ${{ matrix.packages }} + + - name: Generate project files + run: | + mkdir ${{ matrix.build-dir || '.not-used' }} + cd ${{ matrix.build-dir || '.' }} + cmake ${{ matrix.build-src-dir || '.' }} ${{ matrix.cmake-args }} -DCMAKE_BUILD_TYPE=${{ matrix.build-config || 'Release' }} -DBUILD_SHARED_LIBS=OFF -DBUILD_FUZZERS=ON + env: + CC: ${{ matrix.compiler }} + CFLAGS: ${{ matrix.cflags }} + LDFLAGS: ${{ matrix.ldflags }} + CI: true + + - name: Compile source code + run: | + cd ${{ matrix.build-dir || '.' }} + cmake --build . --config ${{ matrix.build-config || 'Release' }} + + - name: Run test cases + run: | + cd ${{ matrix.build-dir || '.' }} + ctest -C Release --output-on-failure --max-width 120 diff -Nru c-blosc-1.17.1+ds1/.github/workflows/fuzz.yml c-blosc-1.21.1+ds2/.github/workflows/fuzz.yml --- c-blosc-1.17.1+ds1/.github/workflows/fuzz.yml 1970-01-01 00:00:00.000000000 +0000 +++ c-blosc-1.21.1+ds2/.github/workflows/fuzz.yml 2021-10-06 08:16:29.000000000 +0000 @@ -0,0 +1,23 @@ +name: CIFuzz +on: [push, pull_request] +jobs: + Fuzzing: + runs-on: ubuntu-latest + steps: + - name: Build Fuzzers + uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@master + with: + oss-fuzz-project-name: 'c-blosc' + dry-run: false + - name: Run Fuzzers + uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@master + with: + oss-fuzz-project-name: 'c-blosc' + fuzz-seconds: 600 + dry-run: false + - name: Upload Crash + uses: actions/upload-artifact@v1 + if: failure() + with: + name: artifacts + path: ./out/artifacts diff -Nru c-blosc-1.17.1+ds1/README_CHUNK_FORMAT.rst c-blosc-1.21.1+ds2/README_CHUNK_FORMAT.rst --- c-blosc-1.17.1+ds1/README_CHUNK_FORMAT.rst 1970-01-01 00:00:00.000000000 +0000 +++ c-blosc-1.21.1+ds2/README_CHUNK_FORMAT.rst 2021-10-06 08:16:29.000000000 +0000 @@ -0,0 +1,97 @@ +Blosc Chunk Format +================== + +The chunk is composed by a header and a blocks / splits section:: + + +---------+--------+---------+ + | header | blocks / splits | + +---------+--------+---------+ + +These are described below. + +The header section +------------------ + +Blosc (as of Version 1.0.0) has the following 16 byte header that stores +information about the compressed buffer:: + + |-0-|-1-|-2-|-3-|-4-|-5-|-6-|-7-|-8-|-9-|-A-|-B-|-C-|-D-|-E-|-F-| + ^ ^ ^ ^ | nbytes | blocksize | cbytes | + | | | | + | | | +--typesize + | | +------flags + | +----------versionlz + +--------------version + +Datatypes of the header entries +------------------------------- + +All entries are little endian. + +:version: + (``uint8``) Blosc format version. +:versionlz: + (``uint8``) Version of the internal compressor used. +:flags and compressor enumeration: + (``bitfield``) The flags of the buffer + + :bit 0 (``0x01``): + Whether the byte-shuffle filter has been applied or not. + :bit 1 (``0x02``): + Whether the internal buffer is a pure memcpy or not. + :bit 2 (``0x04``): + Whether the bit-shuffle filter has been applied or not. + :bit 3 (``0x08``): + Reserved, must be zero. + :bit 4 (``0x10``): + If set, the blocks will not be split in sub-blocks during compression. + :bit 5 (``0x20``): + Part of the enumeration for compressors. + :bit 6 (``0x40``): + Part of the enumeration for compressors. + :bit 7 (``0x80``): + Part of the enumeration for compressors. + + The last three bits form an enumeration that allows to use alternative + compressors. + + :``0``: + ``blosclz`` + :``1``: + ``lz4`` or ``lz4hc`` + :``2``: + ``snappy`` + :``3``: + ``zlib`` + :``4``: + ``zstd`` + +:typesize: + (``uint8``) Number of bytes for the atomic type. +:nbytes: + (``uint32``) Uncompressed size of the buffer (this header is not included). +:blocksize: + (``uint32``) Size of internal blocks. +:cbytes: + (``uint32``) Compressed size of the buffer (including this header). + +The blocks / splits section +--------------------------- + +After the header, there come the blocks / splits section. Blocks are equal-sized parts of the chunk, except for the last block that can be shorter or equal than the rest. + +At the beginning of the blocks section, there come a list of `int32_t bstarts` to indicate where the different encoded blocks starts (counting from the end of this `bstarts` section):: + + +=========+=========+========+=========+ + | bstart0 | bstart1 | ... | bstartN | + +=========+=========+========+=========+ + +Finally, it comes the actual list of compressed blocks / splits data streams. It turns out that a block may optionally (see bit 4 in `flags` above) be further split in so-called splits which are the actual data streams that are transmitted to codecs for compression. If a block is not split, then the split is equivalent to a whole block. Before each split in the list, there is the compressed size of it, expressed as an `int32_t`:: + + +========+========+========+========+========+========+========+ + | csize0 | split0 | csize1 | split1 | ... | csizeN | splitN | + +========+========+========+========+========+========+========+ + + +*Note*: all the integers are stored in little endian. + diff -Nru c-blosc-1.17.1+ds1/README_HEADER.rst c-blosc-1.21.1+ds2/README_HEADER.rst --- c-blosc-1.17.1+ds1/README_HEADER.rst 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/README_HEADER.rst 1970-01-01 00:00:00.000000000 +0000 @@ -1,65 +0,0 @@ -Blosc Header Format -=================== - -Blosc (as of Version 1.0.0) has the following 16 byte header that stores -information about the compressed buffer:: - - |-0-|-1-|-2-|-3-|-4-|-5-|-6-|-7-|-8-|-9-|-A-|-B-|-C-|-D-|-E-|-F-| - ^ ^ ^ ^ | nbytes | blocksize | ctbytes | - | | | | - | | | +--typesize - | | +------flags - | +----------versionlz - +--------------version - -Datatypes of the Header Entries -------------------------------- - -All entries are little endian. - -:version: - (``uint8``) Blosc format version. -:versionlz: - (``uint8``) Version of the internal compressor used. -:flags and compressor enumeration: - (``bitfield``) The flags of the buffer - - :bit 0 (``0x01``): - Whether the byte-shuffle filter has been applied or not. - :bit 1 (``0x02``): - Whether the internal buffer is a pure memcpy or not. - :bit 2 (``0x04``): - Whether the bit-shuffle filter has been applied or not. - :bit 3 (``0x08``): - Reserved, must be zero. - :bit 4 (``0x10``): - If set, the blocks will not be split in sub-blocks during compression. - :bit 5 (``0x20``): - Part of the enumeration for compressors. - :bit 6 (``0x40``): - Part of the enumeration for compressors. - :bit 7 (``0x80``): - Part of the enumeration for compressors. - - The last three bits form an enumeration that allows to use alternative - compressors. - - :``0``: - ``blosclz`` - :``1``: - ``lz4`` or ``lz4hc`` - :``2``: - ``snappy`` - :``3``: - ``zlib`` - :``4``: - ``zstd`` - -:typesize: - (``uint8``) Number of bytes for the atomic type. -:nbytes: - (``uint32``) Uncompressed size of the buffer. -:blocksize: - (``uint32``) Size of internal blocks. -:ctbytes: - (``uint32``) Compressed size of the buffer. diff -Nru c-blosc-1.17.1+ds1/README.md c-blosc-1.21.1+ds2/README.md --- c-blosc-1.17.1+ds1/README.md 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/README.md 2021-10-06 08:16:29.000000000 +0000 @@ -3,10 +3,9 @@ |--------|---------|-----| | Blosc Development Team | blosc@blosc.org | http://www.blosc.org | -| Gitter | Travis CI | Appveyor | NumFOCUS | -|--------|-----------|----------|----------| -| [![Build Status](https://badges.gitter.im/Blosc/c-blosc.svg)](https://gitter.im/Blosc/c-blosc?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) | [![Build Status](https://travis-ci.org/Blosc/c-blosc.svg?branch=master)](https://travis-ci.org/Blosc/c-blosc) | [![Build Status](https://ci.appveyor.com/api/projects/status/3mlyjc1ak0lbkmte?svg=true)](https://ci.appveyor.com/project/FrancescAlted/c-blosc/branch/master) | [![Powered by NumFOCUS](https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A)](http://numfocus.org) | - +| Gitter | GH Actions | NumFOCUS | Code of Conduct | +|--------|------------|----------|-----------------| +| [![Gitter](https://badges.gitter.im/Blosc/c-blosc.svg)](https://gitter.im/Blosc/c-blosc?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) | [![CI CMake](https://github.com/Blosc/c-blosc/workflows/CI%20CMake/badge.svg)](https://github.com/Blosc/c-blosc/actions?query=workflow%3A%22CI+CMake%22) | [![Powered by NumFOCUS](https://img.shields.io/badge/powered%20by-NumFOCUS-orange.svg?style=flat&colorA=E1523D&colorB=007D8A)](http://numfocus.org) | [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg)](code_of_conduct.md) | ## What is it? diff -Nru c-blosc-1.17.1+ds1/RELEASE_NOTES.rst c-blosc-1.21.1+ds2/RELEASE_NOTES.rst --- c-blosc-1.17.1+ds1/RELEASE_NOTES.rst 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/RELEASE_NOTES.rst 2021-10-06 08:16:29.000000000 +0000 @@ -2,6 +2,101 @@ Release notes for C-Blosc =========================== +Changes from 1.21.0 to 1.21.1 +============================= + +* Fix pthread flag when linking on ppc64le. See #318. Thanks to Axel Huebl. + +* Updates in codecs (some bring important performance improvements): + * BloscLZ updated to 2.5.1. + * Zlib updated to 1.2.11 + * Zstd updated to 1.5.0 + + +Changes from 1.20.1 to 1.21.0 +============================= + +* Updated zstd codec to 1.4.8. + +* Updated lz4 codec to 1.9.3. + +* New instructions on how to use the libraries in python-blosc wheels + so as to compile C-Blosc applications. See: + https://github.com/Blosc/c-blosc/blob/master/COMPILING_WITH_WHEELS.rst + + +Changes from 1.20.0 to 1.20.1 +============================= + +* Added `` in vendored zlib 1.2.8 for compatibility with Python 3.8 + in recent Mac OSX. For details, see: + https://github.com/Blosc/python-blosc/issues/229 + + +Changes from 1.19.1 to 1.20.0 +============================= + +* More saftey checks have been implemented so that potential flaws + discovered by new fuzzers in OSS-Fuzzer are fixed now. Thanks to + Nathan Moinvaziri (@nmoinvaz). + +* BloscLZ updated to 2.3.0. Expect better compression ratios for faster + codecs. For details, see our new blog post: + https://blosc.org/posts/beast-release/ + +* Fixed the `_xgetbv()` collision. Thanks to Michał Górny (@mgorny). + +* The chunk format has been fully described so that 3rd party software + may come with a different implementation, but still compatible with + C-Blosc chunks. + + +Changes from 1.19.0 to 1.19.1 +============================= + +- pthread_create() errors are now handled and propagated back to the user. + See https://github.com/Blosc/c-blosc/pull/299. + + +Changes from 1.18.1 to 1.19.0 +============================= + +- The length of automatic blocksizes for fast codecs (lz4, blosclz) has + been incremented quite a bit (up to 256 KB) for better compression ratios. + The performance in modern CPUs (with at least 256 KB in L2 cache) should + be better too (for older CPUs the performance should stay roughly the same). + +- Continuous integration has been migrated to GitHub actions and much + more scenarios are tested (specially linking with external codecs). + Also, a new OSS-Fuzz workflow has been added for increased detection + of possible vulnerabilities. Thanks to Nathan Moinvaziri. + +- For small buffers that cannot be compressed (typically < 128 bytes), + `blosc_compress()` returns now a 0 (cannot compress) instead of a negative + number (internal error). See https://github.com/Blosc/c-blosc/pull/294. + Thanks to @kalvdans for providing the initial patch. + +- blosclz codec updated to 2.1.0. Expect better compression ratios and + performance in a wider variety of scenarios. + +- `blosc_decompress_unsafe()`, `blosc_decompress_ctx_unsafe()` and + `blosc_getitem_unsafe()` have been removed because they are dangerous + and after latest improvements, they should not be used in production. + +- zstd codec updated to 1.4.5. + +- Conan packaging has been deprecated (from now on, we should try + to focus on supporting wheels only). + + +Changes from 1.17.1 to 1.18.1 +============================= + +- Fixed the copy of the leftovers of a chunk when its size is not a + multiple of the typesize. Although this is a very unusual situation, + it can certainly happen (e.g. + https://github.com/Blosc/python-blosc/issues/220). + Changes from 1.17.0 to 1.17.1 ============================= diff -Nru c-blosc-1.17.1+ds1/RELEASING.rst c-blosc-1.21.1+ds2/RELEASING.rst --- c-blosc-1.17.1+ds1/RELEASING.rst 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/RELEASING.rst 2021-10-06 08:16:29.000000000 +0000 @@ -45,7 +45,7 @@ $ cd ../compat $ export LD_LIBRARY_PATH=../build/blosc $ gcc -o filegen filegen.c -L$LD_LIBRARY_PATH -lblosc -I../blosc - $ ./filegen compress lz4 blosc-lz4-1.y.z.cdata + $ ./filegen compress lz4 blosc-1.y.z-lz4.cdata In order to make sure that we are not breaking forward compatibility, link and run the `compat/filegen` utility against different versions of @@ -58,7 +58,13 @@ Then, test the file created with the new version with:: - $ ./filegen decompress blosc-lz4-1.y.z.cdata + $ ./filegen decompress blosc-1.y.z-lz4.cdata + +If that works and you want to keep track of this for future compatibility checks +just add the new file to the suite:: + + $ git add blosc-1.y.z-lz4.cdata + $ git commit -m"Add a new cdata file for compatibility checks" Repeat this for every codec shipped with Blosc (blosclz, lz4, lz4hc, snappy, zlib and zstd). @@ -66,8 +72,9 @@ Tagging ------- -- Create a tag ``X.Y.Z`` from ``master``. Use the next message:: +- Create a tag ``X.Y.Z`` from ``master``:: + $ git switch master $ git tag -a vX.Y.Z -m "Tagging version X.Y.Z" - Push the previous commits and tag to the github repo:: diff -Nru c-blosc-1.17.1+ds1/test_package/CMakeLists.txt c-blosc-1.21.1+ds2/test_package/CMakeLists.txt --- c-blosc-1.17.1+ds1/test_package/CMakeLists.txt 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/test_package/CMakeLists.txt 1970-01-01 00:00:00.000000000 +0000 @@ -1,8 +0,0 @@ -project(PackageTest CXX) -cmake_minimum_required(VERSION 2.8.12) - -include(${CMAKE_BINARY_DIR}/conanbuildinfo.cmake) -conan_basic_setup() - -add_executable(example example.cpp) -target_link_libraries(example ${CONAN_LIBS}) diff -Nru c-blosc-1.17.1+ds1/test_package/conanfile.py c-blosc-1.21.1+ds2/test_package/conanfile.py --- c-blosc-1.17.1+ds1/test_package/conanfile.py 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/test_package/conanfile.py 1970-01-01 00:00:00.000000000 +0000 @@ -1,24 +0,0 @@ -from conans import ConanFile, CMake, tools - - -class CbloscTestConan(ConanFile): - settings = "os", "compiler", "build_type", "arch" - generators = "cmake" - - def build(self): - cmake = CMake(self) - cmake.configure() - cmake.build() - - def imports(self): - self.copy("*.dll", dst="bin", src="bin" ) - self.copy("*.dylib*", dst="bin", src="lib") - self.copy("*.so*", dst="bin", src="lib") - - def test(self): - with tools.chdir("bin"): - if tools.os_info.is_windows: - self.run("example") - else: - prefix = "DYLD_LIBRARY_PATH=." if tools.os_info.is_macos else "" - self.run("%s ./example" % prefix) diff -Nru c-blosc-1.17.1+ds1/test_package/example.cpp c-blosc-1.21.1+ds2/test_package/example.cpp --- c-blosc-1.17.1+ds1/test_package/example.cpp 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/test_package/example.cpp 1970-01-01 00:00:00.000000000 +0000 @@ -1,59 +0,0 @@ -#include -#include - -#define SIZE 100*100*100 - -int main(){ - static float data[SIZE]; - static float data_out[SIZE]; - static float data_dest[SIZE]; - int isize = SIZE*sizeof(float), osize = SIZE*sizeof(float); - int dsize = SIZE*sizeof(float), csize; - int i; - - for(i=0; i %d (%.1fx)\n", isize, csize, (1.*isize) / csize); - - /* Decompress */ - dsize = blosc_decompress(data_out, data_dest, dsize); - if (dsize < 0) { - printf("Decompression error. Error code: %d\n", dsize); - return dsize; - } - - printf("Decompression succesful!\n"); - - /* After using it, destroy the Blosc environment */ - blosc_destroy(); - - for(i=0;i +#include + +#include "blosc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + const char *compressors[] = { "blosclz", "lz4", "lz4hc", "snappy", "zlib", "zstd" }; + int level = 9, filter = BLOSC_BITSHUFFLE, cindex = 0, i = 0; + size_t nbytes, cbytes, blocksize; + void *output, *input; + + blosc_set_nthreads(1); + + if (size > 0) + level = data[0] % (9 + 1); + if (size > 1) + filter = data[1] % (BLOSC_BITSHUFFLE + 1); + if (size > 2) + cindex = data[2]; + + /* Find next available compressor */ + while (blosc_set_compressor(compressors[cindex % 6]) == -1 && i < 6) { + cindex++, i++; + } + if (i == 6) { + /* No compressors available */ + return 0; + } + + if (size > 3 && data[3] % 7 == 0) + blosc_set_blocksize(4096); + + if (size > 4) + blosc_set_splitmode(data[4] % BLOSC_FORWARD_COMPAT_SPLIT + 1); + + output = malloc(size + 1); + if (output == NULL) + return 0; + + if (blosc_compress(level, filter, 1, size, data, output, size) == 0) { + /* Cannot compress src buffer into dest */ + free(output); + return 0; + } + + blosc_cbuffer_sizes(output, &nbytes, &cbytes, &blocksize); + + input = malloc(cbytes); + if (input != NULL) { + blosc_decompress(output, input, cbytes); + free(input); + } + + free(output); + + return 0; +} + +#ifdef __cplusplus +} +#endif diff -Nru c-blosc-1.17.1+ds1/tests/fuzz/fuzz_decompress.c c-blosc-1.21.1+ds2/tests/fuzz/fuzz_decompress.c --- c-blosc-1.17.1+ds1/tests/fuzz/fuzz_decompress.c 1970-01-01 00:00:00.000000000 +0000 +++ c-blosc-1.21.1+ds2/tests/fuzz/fuzz_decompress.c 2021-10-06 08:16:29.000000000 +0000 @@ -0,0 +1,41 @@ +#include +#include + +#include "blosc.h" + +#ifdef __cplusplus +extern "C" { +#endif + +int LLVMFuzzerTestOneInput(const uint8_t *data, size_t size) { + size_t nbytes, cbytes, blocksize; + void *output; + + if (size < BLOSC_MIN_HEADER_LENGTH) { + return 0; + } + + blosc_cbuffer_sizes(data, &nbytes, &cbytes, &blocksize); + if (cbytes != size) { + return 0; + } + if (nbytes == 0) { + return 0; + } + + if (blosc_cbuffer_validate(data, size, &nbytes) != 0) { + /* Unexpected nbytes specified in blosc header */ + return 0; + } + + output = malloc(cbytes); + if (output != NULL) { + blosc_decompress(data, output, cbytes); + free(output); + } + return 0; +} + +#ifdef __cplusplus +} +#endif diff -Nru c-blosc-1.17.1+ds1/tests/fuzz/standalone.c c-blosc-1.21.1+ds2/tests/fuzz/standalone.c --- c-blosc-1.17.1+ds1/tests/fuzz/standalone.c 1970-01-01 00:00:00.000000000 +0000 +++ c-blosc-1.21.1+ds2/tests/fuzz/standalone.c 2021-10-06 08:16:29.000000000 +0000 @@ -0,0 +1,44 @@ +#include +#include +#include + +extern int LLVMFuzzerTestOneInput(const unsigned char *data, size_t size); + +int main(int argc, char **argv) { + int i; + fprintf(stderr, "Running %d inputs\n", argc - 1); + + for (i = 1; i < argc; i++) { + size_t len, err, n_read = 0; + unsigned char *buf; + FILE *f = NULL; + + f = fopen(argv[i], "rb+"); + if (f == NULL) { + /* Failed to open this file: it may be a directory. */ + fprintf(stderr, "Skipping: %s\n", argv[i]); + continue; + } + fprintf(stderr, "Running: %s %s\n", argv[0], argv[i]); + + fseek(f, 0, SEEK_END); + len = ftell(f); + fseek(f, 0, SEEK_SET); + + buf = (unsigned char *)malloc(len); + if (buf != NULL) { + n_read = fread(buf, 1, len, f); + assert(n_read == len); + LLVMFuzzerTestOneInput(buf, len); + free(buf); + } + + err = fclose(f); + assert(err == 0); + (void)err; + + fprintf(stderr, "Done: %s: (%d bytes)\n", argv[i], (int)n_read); + } + + return 0; +} diff -Nru c-blosc-1.17.1+ds1/tests/test_bitshuffle_leftovers.c c-blosc-1.21.1+ds2/tests/test_bitshuffle_leftovers.c --- c-blosc-1.17.1+ds1/tests/test_bitshuffle_leftovers.c 1970-01-01 00:00:00.000000000 +0000 +++ c-blosc-1.21.1+ds2/tests/test_bitshuffle_leftovers.c 2021-10-06 08:16:29.000000000 +0000 @@ -0,0 +1,140 @@ +/********************************************************************* + Blosc - Blocked Shuffling and Compression Library + + Unit test for the bitshuffle with blocks that are not aligned. + See https://github.com/Blosc/python-blosc/issues/220 + Probably related: https://github.com/Blosc/c-blosc/issues/240 + + Creation date: 2020-02-18 + Author: Francesc Alted + + See LICENSES/BLOSC.txt for details about copyright and rights to use. + **********************************************************************/ + +#include "test_common.h" + + +static int test_roundtrip_bitshuffle8(int size, void *data, void *data_out, void *data_dest) { + /* Compress with bitshuffle active */ + int isize = size; + int osize = size + BLOSC_MIN_HEADER_LENGTH; + int csize = blosc_compress(9, BLOSC_BITSHUFFLE, 8, isize, data, data_out, osize); + int dsize; + int exit_code; + FILE *fout = fopen("test-bitshuffle8-nomemcpy.cdata", "w"); + + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 1; + } + else if (csize < 0) { + printf("Compression error. Error code: %d\n", csize); + return csize; + } + printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1.*isize) / csize); + + fwrite(data_out, csize, 1, fout); + fclose(fout); + + /* Decompress */ + dsize = blosc_decompress(data_out, data_dest, isize); + if (dsize < 0) { + printf("Decompression error. Error code: %d\n", dsize); + return dsize; + } + + printf("Decompression succesful!\n"); + + exit_code = memcmp(data, data_dest, size) ? EXIT_FAILURE : EXIT_SUCCESS; + + if (exit_code == EXIT_SUCCESS) + printf("Succesful roundtrip!\n"); + else + printf("Decompressed data differs from original!\n"); + + return exit_code; +} + +static int test_roundtrip_bitshuffle4(int size, void *data, void *data_out, void *data_dest) { + /* Compress with bitshuffle active */ + int isize = size; + int osize = size + BLOSC_MIN_HEADER_LENGTH; + int csize = blosc_compress(9, BLOSC_BITSHUFFLE, 4, isize, data, data_out, osize); + int dsize; + int exit_code; + FILE *fout = fopen("test-bitshuffle4-memcpy.cdata", "w"); + + if (csize == 0) { + printf("Buffer is uncompressible. Giving up.\n"); + return 1; + } + else if (csize < 0) { + printf("Compression error. Error code: %d\n", csize); + return csize; + } + printf("Compression: %d -> %d (%.1fx)\n", isize, csize, (1.*isize) / csize); + + fwrite(data_out, csize, 1, fout); + fclose(fout); + + /* Decompress */ + dsize = blosc_decompress(data_out, data_dest, isize); + if (dsize < 0) { + printf("Decompression error. Error code: %d\n", dsize); + return dsize; + } + + printf("Decompression succesful!\n"); + + exit_code = memcmp(data, data_dest, size) ? EXIT_FAILURE : EXIT_SUCCESS; + if (exit_code == EXIT_SUCCESS) + printf("Succesful roundtrip!\n"); + else + printf("Decompressed data differs from original!\n"); + + return exit_code; +} + +int main() { + /* `size` below is chosen so that it is not divisible by 8 + * (not supported by bitshuffle) and in addition, it is not + * divisible by 8 (typesize) again. + */ + int size = 641091; + int32_t *data = malloc(size); + int32_t *data_out = malloc(size + BLOSC_MIN_HEADER_LENGTH); + int32_t *data_dest = malloc(size); + int result; + int i; + + /* Initialize data */ + for (i = 0; i < size / sizeof(int32_t); i++) { + ((uint32_t*)data)[i] = i; + } + /* leftovers */ + for (i = size / sizeof(int32_t) * sizeof(int32_t); i < size; i++) { + ((uint8_t*)data)[i] = i; + } + + blosc_init(); + blosc_set_nthreads(1); + blosc_set_compressor("lz4"); + printf("Blosc version info: %s (%s)\n", BLOSC_VERSION_STRING, BLOSC_VERSION_DATE); + result = test_roundtrip_bitshuffle4(size, data, data_out, data_dest); + if (result != EXIT_SUCCESS) { + goto fail; + } + result = test_roundtrip_bitshuffle8(size, data, data_out, data_dest); + if (result != EXIT_SUCCESS) { + goto fail; + } + + free(data); + free(data_out); + free(data_dest); + + blosc_destroy(); + + fail: + return result; +} diff -Nru c-blosc-1.17.1+ds1/tests/test_compressor.c c-blosc-1.21.1+ds2/tests/test_compressor.c --- c-blosc-1.17.1+ds1/tests/test_compressor.c 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/tests/test_compressor.c 2021-10-06 08:16:29.000000000 +0000 @@ -151,7 +151,7 @@ int cbytes2; /* Get a compressed buffer */ - blosc_set_compressor("blosclz"); /* avoid lz4 here for now (see #BLOSC_MAX_OVERHEAD8) */ + blosc_set_compressor("lz4"); /* avoid lz4 here for now (see #BLOSC_MAX_OVERHEAD8) */ cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD); mu_assert("ERROR: cbytes is not 0", cbytes < size); @@ -160,8 +160,9 @@ setenv("BLOSC_SHUFFLE", "BITSHUFFLE", 0); cbytes2 = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD); + /* For some reason, shuffle is unreasonably efficient here (much more than bitshuffle) */ mu_assert("ERROR: BLOSC_SHUFFLE=BITSHUFFLE does not work correctly", - cbytes2 < cbytes * 1.5); + cbytes2 < cbytes * 20); /* Reset env var */ unsetenv("BLOSC_SHUFFLE"); diff -Nru c-blosc-1.17.1+ds1/tests/test_maxout.c c-blosc-1.21.1+ds2/tests/test_maxout.c --- c-blosc-1.17.1+ds1/tests/test_maxout.c 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/tests/test_maxout.c 2021-10-06 08:16:29.000000000 +0000 @@ -22,11 +22,24 @@ size_t size = 1000; /* must be divisible by 4 */ +/* Check input size > BLOSC_MAX_BUFFERSIZE */ +static const char *test_input_too_large(void) { + + /* Get a compressed buffer */ + cbytes = blosc_compress(clevel, doshuffle, typesize, BLOSC_MAX_BUFFERSIZE + 1, src, + dest, size + BLOSC_MAX_OVERHEAD - 1); + mu_assert("ERROR: cbytes is not 0", cbytes == 0); + + return 0; +} + + /* Check maxout with maxout < size */ static const char *test_maxout_less(void) { /* Get a compressed buffer */ - cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD - 1); + cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, + size + BLOSC_MAX_OVERHEAD - 1); mu_assert("ERROR: cbytes is not 0", cbytes == 0); return 0; @@ -37,7 +50,8 @@ static const char *test_maxout_less_memcpy(void) { /* Get a compressed buffer */ - cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD - 1); + cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, + size + BLOSC_MAX_OVERHEAD - 1); mu_assert("ERROR: cbytes is not 0", cbytes == 0); return 0; @@ -48,7 +62,8 @@ static const char *test_maxout_equal(void) { /* Get a compressed buffer */ - cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD); + cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, + size + BLOSC_MAX_OVERHEAD); mu_assert("ERROR: cbytes is not correct", cbytes == size + BLOSC_MAX_OVERHEAD); /* Decompress the buffer */ @@ -63,7 +78,8 @@ static const char *test_maxout_equal_memcpy(void) { /* Get a compressed buffer */ - cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD); + cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, + size + BLOSC_MAX_OVERHEAD); mu_assert("ERROR: cbytes is not correct", cbytes == size + BLOSC_MAX_OVERHEAD); /* Decompress the buffer */ @@ -77,7 +93,8 @@ /* Check maxout with maxout > size */ static const char *test_maxout_great(void) { /* Get a compressed buffer */ - cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD + 1); + cbytes = blosc_compress(clevel, doshuffle, typesize, size, src, dest, + size + BLOSC_MAX_OVERHEAD + 1); mu_assert("ERROR: cbytes is not correct", cbytes == size + BLOSC_MAX_OVERHEAD); /* Decompress the buffer */ @@ -91,7 +108,8 @@ /* Check maxout with maxout > size (memcpy version) */ static const char *test_maxout_great_memcpy(void) { /* Get a compressed buffer */ - cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, size + BLOSC_MAX_OVERHEAD + 1); + cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, + size + BLOSC_MAX_OVERHEAD + 1); mu_assert("ERROR: cbytes is not correct", cbytes == size + BLOSC_MAX_OVERHEAD); /* Decompress the buffer */ @@ -104,18 +122,20 @@ /* Check maxout with maxout < BLOSC_MAX_OVERHEAD */ static const char *test_max_overhead(void) { blosc_init(); - cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, BLOSC_MAX_OVERHEAD - 1); - mu_assert("ERROR: cbytes is not correct", cbytes < 0); + cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, + BLOSC_MAX_OVERHEAD - 1); + mu_assert("ERROR: cbytes is not correct", cbytes == 0); blosc_destroy(); blosc_init(); - cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, BLOSC_MAX_OVERHEAD - 2); - mu_assert("ERROR: cbytes is not correct", cbytes < 0); + cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, + BLOSC_MAX_OVERHEAD - 2); + mu_assert("ERROR: cbytes is not correct", cbytes == 0); blosc_destroy(); blosc_init(); cbytes = blosc_compress(0, doshuffle, typesize, size, src, dest, 0); - mu_assert("ERROR: cbytes is not correct", cbytes < 0); + mu_assert("ERROR: cbytes is not correct", cbytes == 0); blosc_destroy(); return 0; @@ -123,6 +143,7 @@ static const char *all_tests(void) { + mu_run_test(test_input_too_large); mu_run_test(test_maxout_less); mu_run_test(test_maxout_less_memcpy); mu_run_test(test_maxout_equal); @@ -175,4 +196,4 @@ blosc_destroy(); return result != 0; -} +} \ No newline at end of file diff -Nru c-blosc-1.17.1+ds1/.travis.yml c-blosc-1.21.1+ds2/.travis.yml --- c-blosc-1.17.1+ds1/.travis.yml 2019-12-12 12:25:17.000000000 +0000 +++ c-blosc-1.21.1+ds2/.travis.yml 1970-01-01 00:00:00.000000000 +0000 @@ -1,80 +0,0 @@ -env: - global: - - CONAN_USERNAME: "francescalted" - - CONAN_LOGIN_USERNAME: "francescalted" - - CONAN_CHANNEL: "stable" - - CONAN_UPLOAD: "https://api.bintray.com/conan/blosc/Conan" - - CONAN_TOTAL_PAGES: 2 - -linux: &linux - os: linux - sudo: required - language: python - python: "3.6" - services: - - docker - -osx: &osx - os: osx - language: generic - -matrix: - include: - - <<: *osx - osx_image: xcode8.3 - env: CONAN_APPLE_CLANG_VERSIONS=8.1 CONAN_CURRENT_PAGE=1 CONAN_RUN_TESTS=1 - - <<: *osx - osx_image: xcode8.3 - env: CONAN_APPLE_CLANG_VERSIONS=8.1 CONAN_CURRENT_PAGE=2 - - - <<: *osx - osx_image: xcode9 - env: CONAN_APPLE_CLANG_VERSIONS=9.0 CONAN_CURRENT_PAGE=1 CONAN_RUN_TESTS=1 - - <<: *osx - osx_image: xcode9 - env: CONAN_APPLE_CLANG_VERSIONS=9.0 CONAN_CURRENT_PAGE=2 - - - <<: *linux - env: CONAN_GCC_VERSIONS=4.9 CONAN_DOCKER_IMAGE=lasote/conangcc49 CONAN_CURRENT_PAGE=1 CONAN_RUN_TESTS=1 - - <<: *linux - env: CONAN_GCC_VERSIONS=4.9 CONAN_DOCKER_IMAGE=lasote/conangcc49 CONAN_CURRENT_PAGE=2 - - - <<: *linux - env: CONAN_GCC_VERSIONS=5 CONAN_DOCKER_IMAGE=lasote/conangcc5 CONAN_CURRENT_PAGE=1 CONAN_RUN_TESTS=1 - - <<: *linux - env: CONAN_GCC_VERSIONS=5 CONAN_DOCKER_IMAGE=lasote/conangcc5 CONAN_CURRENT_PAGE=2 - - - <<: *linux - env: CONAN_GCC_VERSIONS=6 CONAN_DOCKER_IMAGE=lasote/conangcc6 CONAN_CURRENT_PAGE=1 CONAN_RUN_TESTS=1 - - <<: *linux - env: CONAN_GCC_VERSIONS=6 CONAN_DOCKER_IMAGE=lasote/conangcc6 CONAN_CURRENT_PAGE=2 - - - <<: *linux - env: CONAN_GCC_VERSIONS=7 CONAN_DOCKER_IMAGE=lasote/conangcc7 CONAN_CURRENT_PAGE=1 CONAN_RUN_TESTS=1 - - <<: *linux - env: CONAN_GCC_VERSIONS=7 CONAN_DOCKER_IMAGE=lasote/conangcc7 CONAN_CURRENT_PAGE=2 - - - <<: *linux - env: CONAN_CLANG_VERSIONS=3.9 CONAN_DOCKER_IMAGE=lasote/conanclang39 CONAN_CURRENT_PAGE=1 CONAN_RUN_TESTS=1 - - <<: *linux - env: CONAN_CLANG_VERSIONS=3.9 CONAN_DOCKER_IMAGE=lasote/conanclang39 CONAN_CURRENT_PAGE=2 - - - <<: *linux - env: CONAN_CLANG_VERSIONS=4.0 CONAN_DOCKER_IMAGE=lasote/conanclang40 CONAN_CURRENT_PAGE=1 CONAN_RUN_TESTS=1 - - <<: *linux - env: CONAN_CLANG_VERSIONS=4.0 CONAN_DOCKER_IMAGE=lasote/conanclang40 CONAN_CURRENT_PAGE=2 - -before_script: - - if [ "$TRAVIS_OS_NAME" == "osx" ]; then - brew update; - brew upgrade pyenv; - pyenv install 3.6.6; - eval "$(pyenv init -)"; - pyenv global 3.6.6; - python3 --version; - pip3 install --upgrade pip; - fi - - pip3 install conan_package_tools==0.19.3 - -script: - - python3 build.py