lto: support --jobserver-style=fifo for recent GNU make

Message ID e2ff3cc9-c6ea-ffb1-dada-e93e3dac51e9@suse.cz
State New, archived
Headers
Series lto: support --jobserver-style=fifo for recent GNU make |

Commit Message

Martin Liška Aug. 4, 2022, 8:57 a.m. UTC
  After a long time, GNU make has finally implemented named pipes when
it comes to --jobserver-auth. The traditional approach are
provided opened file descriptors that causes troubles:
https://savannah.gnu.org/bugs/index.php?57242

GNU make commit:
https://git.savannah.gnu.org/cgit/make.git/commit/?id=7ad2593b2d2bb5b9332f4444d8bf93ac6f958bc6

I tested that locally with TOT GNU make and it works:

$ cat Makefile
all:
	g++ tramp3d-v4.ii -c -flto -O2
	g++ tramp3d-v4.o -flto=jobserver

$ MAKE=/tmp/bin/bin/make /tmp/bin/bin/make -j16 --jobserver-style=fifo
g++ tramp3d-v4.ii -c -flto -O2
g++ tramp3d-v4.o -flto=jobserver
(ltrans run in parallel)

Ready to be installed after tests?
Martin

gcc/ChangeLog:

	* gcc.cc (driver::detect_jobserver): Support --jobserver-style=fifo.
	* lto-wrapper.cc (jobserver_active_p): Likewise.
---
  gcc/gcc.cc         | 15 ++++++++++++---
  gcc/lto-wrapper.cc | 20 +++++++++++++++-----
  2 files changed, 27 insertions(+), 8 deletions(-)
  

Comments

Richard Biener Aug. 5, 2022, 10:58 a.m. UTC | #1
On Thu, Aug 4, 2022 at 10:57 AM Martin Liška <mliska@suse.cz> wrote:
>
> After a long time, GNU make has finally implemented named pipes when
> it comes to --jobserver-auth. The traditional approach are
> provided opened file descriptors that causes troubles:
> https://savannah.gnu.org/bugs/index.php?57242
>
> GNU make commit:
> https://git.savannah.gnu.org/cgit/make.git/commit/?id=7ad2593b2d2bb5b9332f4444d8bf93ac6f958bc6
>
> I tested that locally with TOT GNU make and it works:
>
> $ cat Makefile
> all:
>         g++ tramp3d-v4.ii -c -flto -O2
>         g++ tramp3d-v4.o -flto=jobserver
>
> $ MAKE=/tmp/bin/bin/make /tmp/bin/bin/make -j16 --jobserver-style=fifo
> g++ tramp3d-v4.ii -c -flto -O2
> g++ tramp3d-v4.o -flto=jobserver
> (ltrans run in parallel)
>
> Ready to be installed after tests?

LGTM.

Thanks,
Richard.

> Martin
>
> gcc/ChangeLog:
>
>         * gcc.cc (driver::detect_jobserver): Support --jobserver-style=fifo.
>         * lto-wrapper.cc (jobserver_active_p): Likewise.
> ---
>   gcc/gcc.cc         | 15 ++++++++++++---
>   gcc/lto-wrapper.cc | 20 +++++++++++++++-----
>   2 files changed, 27 insertions(+), 8 deletions(-)
>
> diff --git a/gcc/gcc.cc b/gcc/gcc.cc
> index 5cbb38560b2..c98407fe03d 100644
> --- a/gcc/gcc.cc
> +++ b/gcc/gcc.cc
> @@ -9182,15 +9182,24 @@ driver::detect_jobserver () const
>     const char *makeflags = env.get ("MAKEFLAGS");
>     if (makeflags != NULL)
>       {
> -      const char *needle = "--jobserver-auth=";
> -      const char *n = strstr (makeflags, needle);
> +      /* Traditionally, GNU make uses opened pieps for jobserver-auth,
> +        e.g. --jobserver-auth=3,4.  */
> +      const char *pipe_needle = "--jobserver-auth=";
> +
> +      /* Starting with GNU make 4.4, one can use --jobserver-style=fifo
> +        and then named pipe is used: --jobserver-auth=fifo:/tmp/hcsparta.  */
> +      const char *fifo_needle = "--jobserver-auth=fifo:";
> +      if (strstr (makeflags, fifo_needle) != NULL)
> +       return;
> +
> +      const char *n = strstr (makeflags, pipe_needle);
>         if (n != NULL)
>         {
>           int rfd = -1;
>           int wfd = -1;
>
>           bool jobserver
> -           = (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
> +           = (sscanf (n + strlen (pipe_needle), "%d,%d", &rfd, &wfd) == 2
>                && rfd > 0
>                && wfd > 0
>                && is_valid_fd (rfd)
> diff --git a/gcc/lto-wrapper.cc b/gcc/lto-wrapper.cc
> index 795ab74555c..756350d5ace 100644
> --- a/gcc/lto-wrapper.cc
> +++ b/gcc/lto-wrapper.cc
> @@ -1342,27 +1342,37 @@ static const char *
>   jobserver_active_p (void)
>   {
>     #define JS_PREFIX "jobserver is not available: "
> -  #define JS_NEEDLE "--jobserver-auth="
> +
> +  /* Traditionally, GNU make uses opened pieps for jobserver-auth,
> +     e.g. --jobserver-auth=3,4.  */
> +  #define JS_PIPE_NEEDLE "--jobserver-auth="
> +
> +  /* Starting with GNU make 4.4, one can use --jobserver-style=fifo
> +     and then named pipe is used: --jobserver-auth=fifo:/tmp/hcsparta.  */
> +  #define JS_FIFO_NEEDLE "--jobserver-auth=fifo:"
>
>     const char *makeflags = getenv ("MAKEFLAGS");
>     if (makeflags == NULL)
>       return JS_PREFIX "%<MAKEFLAGS%> environment variable is unset";
>
> -  const char *n = strstr (makeflags, JS_NEEDLE);
> +  if (strstr (makeflags, JS_FIFO_NEEDLE) != NULL)
> +    return NULL;
> +
> +  const char *n = strstr (makeflags, JS_PIPE_NEEDLE);
>     if (n == NULL)
> -    return JS_PREFIX "%<" JS_NEEDLE "%> is not present in %<MAKEFLAGS%>";
> +    return JS_PREFIX "%<" JS_PIPE_NEEDLE "%> is not present in %<MAKEFLAGS%>";
>
>     int rfd = -1;
>     int wfd = -1;
>
> -  if (sscanf (n + strlen (JS_NEEDLE), "%d,%d", &rfd, &wfd) == 2
> +  if (sscanf (n + strlen (JS_PIPE_NEEDLE), "%d,%d", &rfd, &wfd) == 2
>         && rfd > 0
>         && wfd > 0
>         && is_valid_fd (rfd)
>         && is_valid_fd (wfd))
>       return NULL;
>     else
> -    return JS_PREFIX "cannot access %<" JS_NEEDLE "%> file descriptors";
> +    return JS_PREFIX "cannot access %<" JS_PIPE_NEEDLE "%> file descriptors";
>   }
>
>   /* Print link to -flto documentation with a hint message.  */
> --
> 2.37.1
>
  
Martin Liška Aug. 9, 2022, 11:55 a.m. UTC | #2
On 8/5/22 12:58, Richard Biener wrote:
> On Thu, Aug 4, 2022 at 10:57 AM Martin Liška <mliska@suse.cz> wrote:
>>
>> After a long time, GNU make has finally implemented named pipes when
>> it comes to --jobserver-auth. The traditional approach are
>> provided opened file descriptors that causes troubles:
>> https://savannah.gnu.org/bugs/index.php?57242
>>
>> GNU make commit:
>> https://git.savannah.gnu.org/cgit/make.git/commit/?id=7ad2593b2d2bb5b9332f4444d8bf93ac6f958bc6
>>
>> I tested that locally with TOT GNU make and it works:
>>
>> $ cat Makefile
>> all:
>>         g++ tramp3d-v4.ii -c -flto -O2
>>         g++ tramp3d-v4.o -flto=jobserver
>>
>> $ MAKE=/tmp/bin/bin/make /tmp/bin/bin/make -j16 --jobserver-style=fifo
>> g++ tramp3d-v4.ii -c -flto -O2
>> g++ tramp3d-v4.o -flto=jobserver
>> (ltrans run in parallel)
>>
>> Ready to be installed after tests?
> 
> LGTM.

I've got actually a nicer patch set where I also support jobserver for WPA.
I'm going to send it in a separate thread.

Martin

> 
> Thanks,
> Richard.
> 
>> Martin
>>
>> gcc/ChangeLog:
>>
>>         * gcc.cc (driver::detect_jobserver): Support --jobserver-style=fifo.
>>         * lto-wrapper.cc (jobserver_active_p): Likewise.
>> ---
>>   gcc/gcc.cc         | 15 ++++++++++++---
>>   gcc/lto-wrapper.cc | 20 +++++++++++++++-----
>>   2 files changed, 27 insertions(+), 8 deletions(-)
>>
>> diff --git a/gcc/gcc.cc b/gcc/gcc.cc
>> index 5cbb38560b2..c98407fe03d 100644
>> --- a/gcc/gcc.cc
>> +++ b/gcc/gcc.cc
>> @@ -9182,15 +9182,24 @@ driver::detect_jobserver () const
>>     const char *makeflags = env.get ("MAKEFLAGS");
>>     if (makeflags != NULL)
>>       {
>> -      const char *needle = "--jobserver-auth=";
>> -      const char *n = strstr (makeflags, needle);
>> +      /* Traditionally, GNU make uses opened pieps for jobserver-auth,
>> +        e.g. --jobserver-auth=3,4.  */
>> +      const char *pipe_needle = "--jobserver-auth=";
>> +
>> +      /* Starting with GNU make 4.4, one can use --jobserver-style=fifo
>> +        and then named pipe is used: --jobserver-auth=fifo:/tmp/hcsparta.  */
>> +      const char *fifo_needle = "--jobserver-auth=fifo:";
>> +      if (strstr (makeflags, fifo_needle) != NULL)
>> +       return;
>> +
>> +      const char *n = strstr (makeflags, pipe_needle);
>>         if (n != NULL)
>>         {
>>           int rfd = -1;
>>           int wfd = -1;
>>
>>           bool jobserver
>> -           = (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
>> +           = (sscanf (n + strlen (pipe_needle), "%d,%d", &rfd, &wfd) == 2
>>                && rfd > 0
>>                && wfd > 0
>>                && is_valid_fd (rfd)
>> diff --git a/gcc/lto-wrapper.cc b/gcc/lto-wrapper.cc
>> index 795ab74555c..756350d5ace 100644
>> --- a/gcc/lto-wrapper.cc
>> +++ b/gcc/lto-wrapper.cc
>> @@ -1342,27 +1342,37 @@ static const char *
>>   jobserver_active_p (void)
>>   {
>>     #define JS_PREFIX "jobserver is not available: "
>> -  #define JS_NEEDLE "--jobserver-auth="
>> +
>> +  /* Traditionally, GNU make uses opened pieps for jobserver-auth,
>> +     e.g. --jobserver-auth=3,4.  */
>> +  #define JS_PIPE_NEEDLE "--jobserver-auth="
>> +
>> +  /* Starting with GNU make 4.4, one can use --jobserver-style=fifo
>> +     and then named pipe is used: --jobserver-auth=fifo:/tmp/hcsparta.  */
>> +  #define JS_FIFO_NEEDLE "--jobserver-auth=fifo:"
>>
>>     const char *makeflags = getenv ("MAKEFLAGS");
>>     if (makeflags == NULL)
>>       return JS_PREFIX "%<MAKEFLAGS%> environment variable is unset";
>>
>> -  const char *n = strstr (makeflags, JS_NEEDLE);
>> +  if (strstr (makeflags, JS_FIFO_NEEDLE) != NULL)
>> +    return NULL;
>> +
>> +  const char *n = strstr (makeflags, JS_PIPE_NEEDLE);
>>     if (n == NULL)
>> -    return JS_PREFIX "%<" JS_NEEDLE "%> is not present in %<MAKEFLAGS%>";
>> +    return JS_PREFIX "%<" JS_PIPE_NEEDLE "%> is not present in %<MAKEFLAGS%>";
>>
>>     int rfd = -1;
>>     int wfd = -1;
>>
>> -  if (sscanf (n + strlen (JS_NEEDLE), "%d,%d", &rfd, &wfd) == 2
>> +  if (sscanf (n + strlen (JS_PIPE_NEEDLE), "%d,%d", &rfd, &wfd) == 2
>>         && rfd > 0
>>         && wfd > 0
>>         && is_valid_fd (rfd)
>>         && is_valid_fd (wfd))
>>       return NULL;
>>     else
>> -    return JS_PREFIX "cannot access %<" JS_NEEDLE "%> file descriptors";
>> +    return JS_PREFIX "cannot access %<" JS_PIPE_NEEDLE "%> file descriptors";
>>   }
>>
>>   /* Print link to -flto documentation with a hint message.  */
>> --
>> 2.37.1
>>
  

Patch

diff --git a/gcc/gcc.cc b/gcc/gcc.cc
index 5cbb38560b2..c98407fe03d 100644
--- a/gcc/gcc.cc
+++ b/gcc/gcc.cc
@@ -9182,15 +9182,24 @@  driver::detect_jobserver () const
    const char *makeflags = env.get ("MAKEFLAGS");
    if (makeflags != NULL)
      {
-      const char *needle = "--jobserver-auth=";
-      const char *n = strstr (makeflags, needle);
+      /* Traditionally, GNU make uses opened pieps for jobserver-auth,
+	 e.g. --jobserver-auth=3,4.  */
+      const char *pipe_needle = "--jobserver-auth=";
+
+      /* Starting with GNU make 4.4, one can use --jobserver-style=fifo
+	 and then named pipe is used: --jobserver-auth=fifo:/tmp/hcsparta.  */
+      const char *fifo_needle = "--jobserver-auth=fifo:";
+      if (strstr (makeflags, fifo_needle) != NULL)
+	return;
+
+      const char *n = strstr (makeflags, pipe_needle);
        if (n != NULL)
  	{
  	  int rfd = -1;
  	  int wfd = -1;
  
  	  bool jobserver
-	    = (sscanf (n + strlen (needle), "%d,%d", &rfd, &wfd) == 2
+	    = (sscanf (n + strlen (pipe_needle), "%d,%d", &rfd, &wfd) == 2
  	       && rfd > 0
  	       && wfd > 0
  	       && is_valid_fd (rfd)
diff --git a/gcc/lto-wrapper.cc b/gcc/lto-wrapper.cc
index 795ab74555c..756350d5ace 100644
--- a/gcc/lto-wrapper.cc
+++ b/gcc/lto-wrapper.cc
@@ -1342,27 +1342,37 @@  static const char *
  jobserver_active_p (void)
  {
    #define JS_PREFIX "jobserver is not available: "
-  #define JS_NEEDLE "--jobserver-auth="
+
+  /* Traditionally, GNU make uses opened pieps for jobserver-auth,
+     e.g. --jobserver-auth=3,4.  */
+  #define JS_PIPE_NEEDLE "--jobserver-auth="
+
+  /* Starting with GNU make 4.4, one can use --jobserver-style=fifo
+     and then named pipe is used: --jobserver-auth=fifo:/tmp/hcsparta.  */
+  #define JS_FIFO_NEEDLE "--jobserver-auth=fifo:"
  
    const char *makeflags = getenv ("MAKEFLAGS");
    if (makeflags == NULL)
      return JS_PREFIX "%<MAKEFLAGS%> environment variable is unset";
  
-  const char *n = strstr (makeflags, JS_NEEDLE);
+  if (strstr (makeflags, JS_FIFO_NEEDLE) != NULL)
+    return NULL;
+
+  const char *n = strstr (makeflags, JS_PIPE_NEEDLE);
    if (n == NULL)
-    return JS_PREFIX "%<" JS_NEEDLE "%> is not present in %<MAKEFLAGS%>";
+    return JS_PREFIX "%<" JS_PIPE_NEEDLE "%> is not present in %<MAKEFLAGS%>";
  
    int rfd = -1;
    int wfd = -1;
  
-  if (sscanf (n + strlen (JS_NEEDLE), "%d,%d", &rfd, &wfd) == 2
+  if (sscanf (n + strlen (JS_PIPE_NEEDLE), "%d,%d", &rfd, &wfd) == 2
        && rfd > 0
        && wfd > 0
        && is_valid_fd (rfd)
        && is_valid_fd (wfd))
      return NULL;
    else
-    return JS_PREFIX "cannot access %<" JS_NEEDLE "%> file descriptors";
+    return JS_PREFIX "cannot access %<" JS_PIPE_NEEDLE "%> file descriptors";
  }
  
  /* Print link to -flto documentation with a hint message.  */