diff mbox series

eal/unix: support ZSTD compression for firmwares

Message ID 20240508095214.2541115-1-david.marchand@redhat.com (mailing list archive)
State Superseded
Delegated to: Thomas Monjalon
Headers
Series eal/unix: support ZSTD compression for firmwares |

Checks

Context Check Description
ci/checkpatch warning coding style issues
ci/loongarch-compilation success Compilation OK
ci/loongarch-unit-testing success Unit Testing PASS
ci/Intel-compilation success Compilation OK
ci/github-robot: build success github build: passed
ci/iol-mellanox-Performance success Performance Testing PASS
ci/iol-intel-Performance success Performance Testing PASS
ci/iol-abi-testing success Testing PASS
ci/iol-compile-amd64-testing success Testing PASS
ci/iol-sample-apps-testing success Testing PASS
ci/iol-unit-amd64-testing success Testing PASS
ci/iol-unit-arm64-testing success Testing PASS
ci/iol-broadcom-Performance success Performance Testing PASS
ci/iol-intel-Functional success Functional Testing PASS
ci/iol-broadcom-Functional success Functional Testing PASS
ci/iol-compile-arm64-testing success Testing PASS
ci/intel-Functional success Functional PASS
ci/intel-Testing success Testing PASS

Commit Message

David Marchand May 8, 2024, 9:52 a.m. UTC 
  Ubuntu 24.04 started to compress firmwares with ZSTD compression.

Bugzilla ID: 1437

Signed-off-by: David Marchand <david.marchand@redhat.com>
---
 lib/eal/unix/eal_firmware.c | 42 +++++++++++++++++++++++++++++--------
 1 file changed, 33 insertions(+), 9 deletions(-)

Comments

Bruce Richardson May 8, 2024, 10:50 a.m. UTC | #1 
On Wed, May 08, 2024 at 11:52:14AM +0200, David Marchand wrote:
> Ubuntu 24.04 started to compress firmwares with ZSTD compression.
> 
> Bugzilla ID: 1437
> 
> Signed-off-by: David Marchand <david.marchand@redhat.com>
> ---
>  lib/eal/unix/eal_firmware.c | 42 +++++++++++++++++++++++++++++--------
>  1 file changed, 33 insertions(+), 9 deletions(-)
> 
> diff --git a/lib/eal/unix/eal_firmware.c b/lib/eal/unix/eal_firmware.c
> index 1d47e879c8..065e251f9d 100644
> --- a/lib/eal/unix/eal_firmware.c
> +++ b/lib/eal/unix/eal_firmware.c
> @@ -16,6 +16,21 @@
>  #include "eal_firmware.h"
>  #include "eal_private.h"
>  
> +#ifndef RTE_HAS_LIBARCHIVE
> +/* Fake definitions for the compression_algorithms array below. */
> +struct archive;
> +extern int archive_read_support_filter_xz(struct archive *a);
> +extern int archive_read_support_filter_zstd(struct archive *a);
> +#endif
> +

Do these not lead to unresolved symbols on link?

> +static struct {
> +	const char *suffix;
> +	int (*support_callback)(struct archive *a);
> +} compression_algorithms[] = {
> +	{ "xz", archive_read_support_filter_xz, },
> +	{ "zst", archive_read_support_filter_zstd, },
> +};
> +

Rather than defining stubs for these functions from libarchive, can you
just have an empty list if no libarchive?

  struct archive;  /* may need to be #ifdef'ed perhaps? */
  static struct {
  	const char *suffix;
  	int (*support_callback)(struct archive *a);
  } compression_algorithms[] = {
  #ifdef RTE_HAS_LIBARCHIVE
  	{ "xz", archive_read_support_filter_xz, },
  	{ "zst", archive_read_support_filter_zstd, },
  #endif
  };


>  #ifdef RTE_HAS_LIBARCHIVE
>  

<snip>
Bruce Richardson May 8, 2024, 3:01 p.m. UTC | #2 
On Wed, May 08, 2024 at 11:50:02AM +0100, Bruce Richardson wrote:
> On Wed, May 08, 2024 at 11:52:14AM +0200, David Marchand wrote:
> > Ubuntu 24.04 started to compress firmwares with ZSTD compression.
> > 

Minor nit, "firmware" doesn't really have a plural in English. I'd suggest
using "firmware files" here rather than "firmwares".

> > Bugzilla ID: 1437
> > 
> > Signed-off-by: David Marchand <david.marchand@redhat.com>
> > ---
> >  lib/eal/unix/eal_firmware.c | 42 +++++++++++++++++++++++++++++--------
> >  1 file changed, 33 insertions(+), 9 deletions(-)
> > 
> > diff --git a/lib/eal/unix/eal_firmware.c b/lib/eal/unix/eal_firmware.c
> > index 1d47e879c8..065e251f9d 100644
> > --- a/lib/eal/unix/eal_firmware.c
> > +++ b/lib/eal/unix/eal_firmware.c
> > @@ -16,6 +16,21 @@
> >  #include "eal_firmware.h"
> >  #include "eal_private.h"
> >  
> > +#ifndef RTE_HAS_LIBARCHIVE
> > +/* Fake definitions for the compression_algorithms array below. */
> > +struct archive;
> > +extern int archive_read_support_filter_xz(struct archive *a);
> > +extern int archive_read_support_filter_zstd(struct archive *a);
> > +#endif
> > +
> 
> Do these not lead to unresolved symbols on link?
> 
Confirmed; in my tests, I do get unresolved symbols on linking without
libarchive.

On the plus side, this does fix the issues when libarchive-dev package is
installed.

/Bruce
David Marchand May 13, 2024, 9:46 a.m. UTC | #3 
On Wed, May 8, 2024 at 12:50 PM Bruce Richardson
<bruce.richardson@intel.com> wrote:
>
> On Wed, May 08, 2024 at 11:52:14AM +0200, David Marchand wrote:
> > Ubuntu 24.04 started to compress firmwares with ZSTD compression.
> >
> > Bugzilla ID: 1437
> >
> > Signed-off-by: David Marchand <david.marchand@redhat.com>
> > ---
> >  lib/eal/unix/eal_firmware.c | 42 +++++++++++++++++++++++++++++--------
> >  1 file changed, 33 insertions(+), 9 deletions(-)
> >
> > diff --git a/lib/eal/unix/eal_firmware.c b/lib/eal/unix/eal_firmware.c
> > index 1d47e879c8..065e251f9d 100644
> > --- a/lib/eal/unix/eal_firmware.c
> > +++ b/lib/eal/unix/eal_firmware.c
> > @@ -16,6 +16,21 @@
> >  #include "eal_firmware.h"
> >  #include "eal_private.h"
> >
> > +#ifndef RTE_HAS_LIBARCHIVE
> > +/* Fake definitions for the compression_algorithms array below. */
> > +struct archive;
> > +extern int archive_read_support_filter_xz(struct archive *a);
> > +extern int archive_read_support_filter_zstd(struct archive *a);
> > +#endif
> > +
>
> Do these not lead to unresolved symbols on link?

Obviously yes... sorry, I was not thinking right and did really
limited testing before sending (only compiled librte_eal.a).
I'll fix this.


>
> > +static struct {
> > +     const char *suffix;
> > +     int (*support_callback)(struct archive *a);
> > +} compression_algorithms[] = {
> > +     { "xz", archive_read_support_filter_xz, },
> > +     { "zst", archive_read_support_filter_zstd, },
> > +};
> > +
>
> Rather than defining stubs for these functions from libarchive, can you
> just have an empty list if no libarchive?
>
>   struct archive;  /* may need to be #ifdef'ed perhaps? */
>   static struct {
>         const char *suffix;
>         int (*support_callback)(struct archive *a);
>   } compression_algorithms[] = {
>   #ifdef RTE_HAS_LIBARCHIVE
>         { "xz", archive_read_support_filter_xz, },
>         { "zst", archive_read_support_filter_zstd, },
>   #endif
>   };

That's an option, but then we lose a warning message to the user
telling that some compressed files are on the system, and linking with
libarchive could help.

I think I'll keep an array of supported extensions and put explicit
calls to archive_read_support_filter_xz/archive_read_support_filter_zstd
under #ifdef.
diff mbox series

Patch

diff --git a/lib/eal/unix/eal_firmware.c b/lib/eal/unix/eal_firmware.c
index 1d47e879c8..065e251f9d 100644
--- a/lib/eal/unix/eal_firmware.c
+++ b/lib/eal/unix/eal_firmware.c
@@ -16,6 +16,21 @@ 
 #include "eal_firmware.h"
 #include "eal_private.h"
 
+#ifndef RTE_HAS_LIBARCHIVE
+/* Fake definitions for the compression_algorithms array below. */
+struct archive;
+extern int archive_read_support_filter_xz(struct archive *a);
+extern int archive_read_support_filter_zstd(struct archive *a);
+#endif
+
+static struct {
+	const char *suffix;
+	int (*support_callback)(struct archive *a);
+} compression_algorithms[] = {
+	{ "xz", archive_read_support_filter_xz, },
+	{ "zst", archive_read_support_filter_zstd, },
+};
+
 #ifdef RTE_HAS_LIBARCHIVE
 
 struct firmware_read_ctx {
@@ -26,7 +41,7 @@  static int
 firmware_open(struct firmware_read_ctx *ctx, const char *name, size_t blocksize)
 {
 	struct archive_entry *e;
-	int err;
+	unsigned int i;
 
 	ctx->a = archive_read_new();
 	if (ctx->a == NULL)
@@ -35,9 +50,12 @@  firmware_open(struct firmware_read_ctx *ctx, const char *name, size_t blocksize)
 	if (archive_read_support_format_raw(ctx->a) != ARCHIVE_OK)
 		goto error;
 
-	err = archive_read_support_filter_xz(ctx->a);
-	if (err != ARCHIVE_OK && err != ARCHIVE_WARN)
-		goto error;
+	for (i = 0; i < RTE_DIM(compression_algorithms); i++) {
+		int err = compression_algorithms[i].support_callback(ctx->a);
+		if (err != ARCHIVE_OK && err != ARCHIVE_WARN)
+			EAL_LOG(WARNING, "could not initialise libarchive for %s compression",
+				compression_algorithms[i].suffix);
+	}
 
 	if (archive_read_open_filename(ctx->a, name, blocksize) != ARCHIVE_OK)
 		goto error;
@@ -148,16 +166,22 @@  rte_firmware_read(const char *name, void **buf, size_t *bufsz)
 
 	ret = firmware_read(name, buf, bufsz);
 	if (ret < 0) {
-		snprintf(path, sizeof(path), "%s.xz", name);
-		path[PATH_MAX - 1] = '\0';
+		unsigned int i;
+
+		for (i = 0; i < RTE_DIM(compression_algorithms); i++) {
+			snprintf(path, sizeof(path), "%s.%s", name,
+				compression_algorithms[i].suffix);
+			path[PATH_MAX - 1] = '\0';
+			if (access(path, F_OK) != 0)
+				continue;
 #ifndef RTE_HAS_LIBARCHIVE
-		if (access(path, F_OK) == 0) {
 			EAL_LOG(WARNING, "libarchive not linked, %s cannot be decompressed",
 				path);
-		}
 #else
-		ret = firmware_read(path, buf, bufsz);
+			ret = firmware_read(path, buf, bufsz);
 #endif
+			break;
+		}
 	}
 	return ret;
 }