swig_bot remote path connection / preliminary implementation.

With this patch, the client will package up all the required
inputs into a compressed zip file, establish a connection to the
server, send the input to the server, and wait for the server to
send a response (in this case the response is just echoed back to
the client).

This gets the network communication in place, and in a subsequent
patch I will follow up with the code that actually runs swig on
the server and sends back the output instead of echoing back the
input.

llvm-svn: 254023

lldb/packages/Python/lldbsuite/support/sockutil.py

@@ -0,0 +1,23 @@
+"""
+                     The LLVM Compiler Infrastructure
+
+This file is distributed under the University of Illinois Open Source
+License. See LICENSE.TXT for details.
+
+Helper functions for working with sockets.
+"""
+
+# Python modules:
+import io
+import socket
+
+# LLDB modules
+import use_lldb_suite
+
+def recvall(sock, size):
+    bytes = io.BytesIO()
+    while size > 0:
+        this_result = sock.recv(size)
+        bytes.write(this_result)
+        size -= len(this_result)
+    return bytes.getvalue()

lldb/scripts/swig_bot.py

@@ -1,11 +1,63 @@
 #!/usr/bin/env python
 
+"""
+SWIG generation top-level script.  Supports both local and remote generation
+of SWIG bindings for multiple languages.
+"""
+
 # Python modules
+import argparse
+import logging
 import sys
+import traceback
 
 # LLDB modules
 import use_lldb_suite
 
+def process_args(args):
+    parser = argparse.ArgumentParser(
+        description='Run swig-bot client or server.')
+
+    # Arguments to control whether swig-bot runs as a client or server.
+    parser.add_argument(
+        "--mode",
+        required=True,
+        choices=["client", "server"],
+        help="Run swig_bot in either client or server mode.")
+
+    # Arguments to control logging verbosity.
+    parser.add_argument(
+        "--verbose", "-v",
+        action="store_true",
+        default=False,
+        help="Increase logging verbosity level.")
+
+    (options, remaining) = parser.parse_known_args(args)
+    # Set logging level.
+    if options.verbose:
+        log_level = logging.DEBUG
+    else:
+        log_level = logging.NOTSET
+    logging.basicConfig(level=log_level)
+    logging.info("logging is using level: %d", log_level)
+
+    return (options, remaining)
+
 if __name__ == "__main__":
-    from swig_bot_lib import client
-    client.run(sys.argv[1:])
+    (options, remaining) = process_args(sys.argv[1:])
+    try:
+        if options.mode == "client":
+            logging.info("Running swig_bot in client mode")
+            from swig_bot_lib import client
+            client.run(remaining)
+        elif options.mode == "server":
+            logging.info("Running swig_bot in server mode")
+            from swig_bot_lib import server
+            server.run(remaining)
+        else:
+            logging.error("Unknown mode specified.  Expected client or server.")
+            sys.exit(-1)
+    except Exception as e:
+        error = traceback.format_exc()
+        logging.error("An error occurred running swig-bot.")
+        logging.error(error)

lldb/scripts/swig_bot_lib/client.py

@@ -1,5 +1,10 @@
 #!/usr/bin/env python
 
+"""
+SWIG generation client.  Supports both local and remote generation of SWIG
+bindings for multiple languages.
+"""
+
 # Future imports
 from __future__ import absolute_import
 from __future__ import print_function
@@ -8,11 +13,20 @@ from __future__ import print_function
 import argparse
 import logging
 import os
+import socket
+import struct
 import sys
 
 # LLDB modules
 import use_lldb_suite
 from lldbsuite.support import fs
+from lldbsuite.support import sockutil
+
+# package imports
+from . import local
+
+default_ip = "127.0.0.1"
+default_port = 8537
 
 def process_args(args):
     """Returns options processed from the provided command line.
@@ -20,9 +34,14 @@ def process_args(args):
     @param args the command line to process.
     """
 
+    # A custom action used by the --local command line option.  It can be
+    # used with either 0 or 1 argument.  If used with 0 arguments, it
+    # searches for a copy of swig located on the physical machine.  If
+    # used with 1 argument, the argument is the path to a swig executable.
     class FindLocalSwigAction(argparse.Action):
         def __init__(self, option_strings, dest, **kwargs):
-            super(FindLocalSwigAction, self).__init__(option_strings, dest, nargs='?', **kwargs)
+            super(FindLocalSwigAction, self).__init__(
+                option_strings, dest, nargs='?', **kwargs)
         def __call__(self, parser, namespace, values, option_string=None):
             swig_exe = None
             if values is None:
@@ -31,17 +50,33 @@ def process_args(args):
                 swig_exe = values
             setattr(namespace, self.dest, os.path.normpath(swig_exe))
 
+    # A custom action used by the --remote command line option.  It can be
+    # used with either 0 or 1 arguments.  If used with 0 arguments it chooses
+    # a default connection string.  If used with one argument it is a string
+    # of the form `ip_address[:port]`.  If the port is unspecified, the
+    # default port is used.
+    class RemoteIpAction(argparse.Action):
+        def __init__(self, option_strings, dest, **kwargs):
+            super(RemoteIpAction, self).__init__(
+                option_strings, dest, nargs='?', **kwargs)
+        def __call__(self, parser, namespace, values, option_string=None):
+            ip_port = None
+            if values is None:
+                ip_port = (default_ip, default_port)
+            else:
+                result = values.split(':')
+                if len(result)==1:
+                    ip_port = (result[0], default_port)
+                elif len(result)==2:
+                    ip_port = (result[0], int(result[1]))
+                else:
+                    raise ValueError("Invalid connection string")
+            setattr(namespace, self.dest, ip_port)
+
     # Setup the parser arguments that are accepted.
     parser = argparse.ArgumentParser(
         description='Generate SWIG bindings.')
 
-    # Arguments to control logging verbosity.
-    parser.add_argument(
-        "--verbose", "-v",
-        action="store_true",
-        default=False,
-        help="Increase logging verbosity level.")
-
     parser.add_argument(
         "--local",
         action=FindLocalSwigAction,
@@ -52,7 +87,7 @@ def process_args(args):
 
     parser.add_argument(
         "--remote",
-        action="store",
+        action=RemoteIpAction,
         help=(
             "Use the given connection string to connect to a remote "
             "generation service"))
@@ -85,24 +120,42 @@ def process_args(args):
         logging.error("Must specify either --local or --remote")
         sys.exit(-3)
 
-    # Set logging level based on verbosity count.
-    if options.verbose:
-        log_level = logging.DEBUG
-    else:
-        log_level = logging.NOTSET
-    logging.basicConfig(level=log_level)
-    logging.info("logging is using level: %d", log_level)
-
     return options
 
+def establish_remote_connection(ip_port):
+    logging.debug("Creating socket...")
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+    logging.info("Connecting to server {} on port {}"
+                 .format(ip_port[0], ip_port[1]))
+    s.connect(ip_port)
+    logging.info("Connection established...")
+    return s
+
+def transmit_data(connection, packed_input):
+    logging.info("Sending {} bytes of compressed data."
+                 .format(len(packed_input)))
+    connection.sendall(struct.pack("!I", len(packed_input)))
+    connection.sendall(packed_input)
+    logging.info("Awaiting response.")
+    response_len = struct.unpack("!I", sockutil.recvall(connection, 4))[0]
+    logging.debug("Expecting {} byte response".format(response_len))
+    response = sockutil.recvall(connection, response_len)
+    return response
 
 def run(args):
     options = process_args(args)
 
     if options.remote is None:
+        logging.info("swig bot client using local swig installation at '{}'"
+                     .format(options.swig_executable))
         if not os.path.isfile(options.swig_executable):
-            logging.error("Swig executable '%s' does not exist." % options.swig_executable)
-        from . import local
+            logging.error("Swig executable '{}' does not exist."
+                          .format(options.swig_executable))
         local.generate(options)
     else:
-        logging.error("Remote path is not yet implemented!")
+        logging.info("swig bot client using remote generation with server '{}'"
+                     .format(options.remote))
+        packed_input = local.pack_input(options)
+        connection = establish_remote_connection(options.remote)
+        response = transmit_data(connection, packed_input)
+        logging.debug("Received {} byte response.".format(len(response)))

lldb/scripts/swig_bot_lib/local.py

@@ -1,5 +1,10 @@
 #!/usr/bin/env python
 
+"""
+Shared functionality used by `client` and `server` when generating or preparing
+to generate SWIG on the local machine.
+"""
+
 # Future imports
 from __future__ import absolute_import
 from __future__ import print_function
@@ -7,14 +12,57 @@ from __future__ import print_function
 # Python modules
 import argparse
 import imp
+import io
 import logging
 import os
 import subprocess
 import sys
+import zipfile
 
 # LLDB modules
 import use_lldb_suite
 
+def pack_input(options):
+    logging.info("Creating input file package...")
+    zip_data = io.BytesIO()
+    zip_file = None
+    try:
+        # It's possible that a custom-built interpreter will not have the
+        # standard zlib module.  If so, we can only store, not compress.  By
+        # try to compress since we usually have a standard Python distribution.
+        zip_file = zipfile.ZipFile(zip_data, mode='w',
+                                   compression=zipfile.ZIP_DEFLATED)
+    except RuntimeError:
+        zip_file = zipfile.ZipFile(zip_data, mode='w',
+                                   compression=zipfile.ZIP_STORED)
+
+    filters = [("include/lldb", ".h"),
+               ("scripts", ".swig"),
+               ("scripts/Python", ".swig"),
+               ("scripts/interface", ".i")]
+    def filter_func(t):
+        subfolder = t[0]
+        ext = t[1]
+        full_path = os.path.normpath(os.path.join(options.src_root, subfolder))
+        candidates = [os.path.normpath(os.path.join(full_path, f))
+                      for f in os.listdir(full_path)]
+        actual = filter(
+            lambda f : os.path.isfile(f) and os.path.splitext(f)[1] == ext,
+            candidates)
+        return (subfolder, map(lambda f : os.path.basename(f), actual))
+    archive_entries = map(filter_func, filters)
+
+    for entry in archive_entries:
+        subfolder = entry[0]
+        files = entry[1]
+        for file in files:
+            relative_path = os.path.normpath(os.path.join(subfolder, file))
+            full_path = os.path.normpath(
+                os.path.join(options.src_root, relative_path))
+            logging.info("{} -> {}".format(full_path, relative_path))
+            zip_file.write(full_path, relative_path)
+    return zip_data.getvalue()
+
 def generate(options):
     include_folder = os.path.join(options.src_root, "include")
     in_file = os.path.join(options.src_root, "scripts", "lldb.swig")
@@ -43,7 +91,8 @@ def generate(options):
             in_file
         ])
 
-        logging.info("generating swig {} bindings into {}".format(lang, out_dir))
+        logging.info("generating swig {} bindings into {}"
+                     .format(lang, out_dir))
         logging.debug("swig command line: {}".format(swig_command))
         try:
             # Execute swig
@@ -54,5 +103,6 @@ def generate(options):
             if swig_output is not None and len(swig_output) > 0:
                 logging.info("swig output: %s", swig_output)
         except subprocess.CalledProcessError as e:
-            logging.error("An error occurred executing swig.  returncode={}".format(e.returncode))
+            logging.error("An error occurred executing swig.  returncode={}"
+                          .format(e.returncode))
             logging.error(e.output)

lldb/scripts/swig_bot_lib/server.py

@@ -1,6 +1,80 @@
 #!/usr/bin/env python
 
+"""
+SWIG generation server.  Listens for connections from swig generation clients
+and runs swig in the requested fashion, sending back the results.
+"""
+
 # Future imports
 from __future__ import absolute_import
 from __future__ import print_function
 
+# Python modules
+import argparse
+import logging
+import os
+import socket
+import struct
+import sys
+import traceback
+
+# LLDB modules
+import use_lldb_suite
+from lldbsuite.support import sockutil
+
+# package imports
+from . import local
+
+default_port = 8537
+
+def process_args(args):
+    # Setup the parser arguments that are accepted.
+    parser = argparse.ArgumentParser(description='SWIG generation server.')
+
+    parser.add_argument(
+        "--port",
+        action="store",
+        default=default_port,
+        help=("The local port to bind to"))
+
+    # Process args.
+    return parser.parse_args(args)
+
+def initialize_listening_socket(options):
+    logging.debug("Creating socket...")
+    s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+
+    logging.info("Binding to ip address '', port {}".format(options.port))
+    s.bind(('', options.port))
+
+    logging.debug("Putting socket in listen mode...")
+    s.listen()
+    return s
+
+def accept_once(sock, options):
+    logging.debug("Waiting for connection...")
+    client, addr = sock.accept()
+    logging.info("Received connection from {}".format(addr))
+    data_size = struct.unpack("!I", sockutil.recvall(client, 4))[0]
+    logging.debug("Expecting {} bytes of data from client".format(data_size))
+    data = sockutil.recvall(client, data_size)
+    logging.info("Received {} bytes of data from client".format(len(data)))
+
+    logging.info("Sending {} byte response".format(len(data)))
+    client.sendall(struct.pack("!I", len(data)))
+    client.sendall(data)
+
+def accept_loop(sock, options):
+    while True:
+        try:
+            accept_once(sock, options)
+        except Exception as e:
+            error = traceback.format_exc()
+            logging.error("An error occurred while processing the connection.")
+            logging.error(error)
+
+def run(args):
+    options = process_args(args)
+    sock = initialize_listening_socket(options)
+    accept_loop(sock, options)
+    return options