Import of the watch repository from Pebble

tools/log_hashing/__init__.py

@@ -0,0 +1,14 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+

tools/log_hashing/check_elf_log_strings.py

@@ -0,0 +1,151 @@
+#!/usr/bin/env python
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import re
+import sys
+import argparse
+
+from elftools.elf.elffile import ELFFile
+from newlogging import get_log_dict_from_file
+
+
+FORMAT_SPECIFIER_REGEX = (r"(?P<flags>[-+ #0])?(?P<width>\*|\d*)?(?P<precision>\.\d+|\.\*)?"
+                          "(?P<length>hh|h|l|ll|j|z|t|L)?(?P<specifier>[diuoxXfFeEgGaAcspn%])")
+FORMAT_SPECIFIER_PATTERN = re.compile(FORMAT_SPECIFIER_REGEX)
+
+FLOAT_SPECIFIERS = "fFeEgGaA"
+LENGTH_64 = ['j', 'll', 'L']   # 'l', 'z', and 't' sizes confirmed to be 32 bits in logging.c
+
+LOG_LEVELS = [0, 1, 50, 100, 200, 255]
+
+def check_elf_log_strings(filename):
+    """ Top Level API """
+    log_dict = get_log_dict_from_file(filename)
+    if not log_dict:
+        return False, ['Unable to get log strings']
+
+    return check_dict_log_strings(log_dict)
+
+
+def check_dict_log_strings(log_dict):
+    """ Return complete error string rather than raise an exception on the first. """
+    output = []
+
+    for log_line in log_dict.itervalues():
+        # Skip build_id and new_logging_version keys
+        if 'file' not in log_line:
+            continue
+
+        file_line = ':'.join(log_line[x] for x in ['file', 'line', 'msg'])
+
+        # Make sure that 'level' is being generated correctly
+        if 'level' in log_line:
+            if not log_line['level'].isdigit():
+                output.append("'{}' PBL_LOG contains a non-constant LOG_LEVEL_ value '{}'".
+                              format(file_line, log_line['level']))
+                break
+            elif int(log_line['level']) not in LOG_LEVELS:
+                output.append("'{}' PBL_LOG contains a non-constant LOG_LEVEL_ value '{}'".
+                              format(file_line, log_line['level']))
+                break
+
+        # Make sure that '`' isn't anywhere in the string
+        if '`' in log_line['msg']:
+            output.append("'{}' PBL_LOG contains '`'".format(file_line))
+
+        # Now check the fmt string rules:
+
+        # To ensure that we capture every %, find the '%' chars and then match on the remaining
+        # string until we're done
+        offset = 0
+        num_conversions = 0
+        num_str_conversions = 0
+
+        while True:
+            offset = log_line['msg'].find('%', offset)
+            if offset == -1:
+                break
+
+            # Match starting immediately after the '%'
+            match = FORMAT_SPECIFIER_PATTERN.match(log_line['msg'][offset + 1:])
+            if not match:
+                output.append("'{}' PBL_LOG contains unknown format specifier".format(file_line))
+                break
+
+            num_conversions += 1
+
+            # RULE: no % literals.
+            if match.group('specifier') == '%':
+                output.append("'{}' PBL_LOG contains '%%'".format(file_line))
+                break
+
+            # RULE: no 64 bit values.
+            if match.group('length') in LENGTH_64:
+                output.append("'{}' PBL_LOG contains 64 bit value".format(file_line))
+                break
+
+            # RULE: no floats. VarArgs promotes to 64 bits, so this won't work, either
+            if match.group('specifier') in FLOAT_SPECIFIERS:
+                output.append("'{}' PBL_LOG contains floating point specifier".
+                              format(file_line))
+                break
+
+            # RULE: no flagged or formatted string conversions
+            if match.group('specifier') == 's':
+                num_str_conversions += 1
+                if match.group('flags') or match.group('width') or match.group('precision'):
+                    output.append("'{}' PBL_LOG contains a formatted string conversion".
+                                  format(file_line))
+                    break
+
+            # RULE: no dynamic width specifiers. I.e., no * or .* widths. '.*' is already covered
+            #       above -- .* specifies precision for floats and # chars for strings. * remains.
+            if '*' in match.group('width'):
+                output.append("'{}' PBL_LOG contains a dynamic width".format(file_line))
+                break
+
+            # Consume this match by updating our offset
+            for text in match.groups():
+                if text:
+                    offset += len(text)
+
+        # RULE: maximum of 7 format conversions
+        if num_conversions > 7:
+            output.append("'{}' PBL_LOG contains more than 7 format conversions".format(file_line))
+
+        # RULE: maximum of 2 string conversions
+        if num_str_conversions > 2:
+            output.append("'{}' PBL_LOG contains more than 2 string conversions".format(file_line))
+
+    if output:
+        output.insert(0, 'NewLogging String Error{}:'.format('s' if len(output) > 1 else ''))
+        output.append("See https://pebbletechnology.atlassian.net/wiki/display/DEV/New+Logging "
+                      "for help")
+
+    return '\n'.join(output)
+
+
+if __name__ == '__main__':
+    parser = argparse.ArgumentParser(description='Check .elf log strings for errors')
+
+    parser.add_argument('elf_path', help='path to tintin_fw.elf to check')
+    args = parser.parse_args()
+
+    output = check_elf_log_strings(args.elf_path)
+
+    if output:
+        print output
+        sys.exit(1)

tools/log_hashing/dehash.py

@@ -0,0 +1,225 @@
+#! /usr/bin/env python
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+import argparse
+import json
+import logging
+import os
+import requests
+import sys
+import zipfile
+
+import logdehash
+import newlogging
+
+
+DICT_FIRMWARE = 'build/src/fw/loghash_dict.json'
+DICT_PRF = 'build/prf/src/fw/loghash_dict.json'
+
+BUILD_ID_STR = 'BUILD ID: '
+HASH_STR_LEN = 40
+
+
+SETTINGS = {
+    # Hagen Daas stuff:
+    'files': 'https://files.pebblecorp.com/dict/',
+
+    # Go to https://auth.pebblecorp.com/show to get this value:
+    'hd_session': None,
+}
+
+
+class AuthException(Exception):
+    pass
+
+
+def load_user_settings():
+    settings_path = '~/.triage'
+    try:
+        user_settings_file = open(os.path.expanduser(settings_path), 'rb')
+        user_settings = json.load(user_settings_file)
+    except IOError as e:
+        if e.errno == 2:
+            logging.error("""Please create %s with credentials: """
+                          """'{ "user": "$USER", "password": "$PASSWORD" }'""",
+                          settings_path)
+        return
+    SETTINGS.update(user_settings)
+
+    if not SETTINGS["hd_session"]:
+        msg = "Missing 'hd_session' token in user settings.\n" \
+              "1. Get the cookie from https://auth.pebblecorp.com/show\n" \
+              "2. Add as value with key 'hd_session' to %s" % settings_path
+        logging.error(msg)
+        sys.exit(-1)
+
+
+def get_loghash_dict_from_hagen_daas_files(hash):
+    load_user_settings()
+
+    url = SETTINGS['files'] + hash
+    r = requests.get(url, headers={'Cookie': 'hd_session=%s' % SETTINGS['hd_session']})
+    if (r.status_code > 400):
+        r.raise_for_status()
+    if "accounts.google.com" in r.url:
+        raise AuthException("Not authenticated, see instructions at the top of %s" %
+                            "https://pebbletechnology.atlassian.net/wiki/"
+                            "display/DEV/Quickly+triaging+JIRA+FW+issues+with+pbldebug")
+    return r.text
+
+
+class Log(object):
+    def __init__(self, output=False):
+        self.output = output
+
+    def setOutput(self, output):
+        self.output = output
+
+    def debug(self, format, *args):
+        if self.output:
+            sys.stderr.write(format % args)
+            sys.stderr.write("\r\n")
+
+
+def get_dict_from_pbz(filename):
+    if zipfile.is_zipfile(filename):
+        with zipfile.ZipFile(filename) as dict_zip:
+            return dict_zip.read('loghash_dict.json')
+    return None
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Dehash a log',
+                                     formatter_class=argparse.RawTextHelpFormatter,
+                                     epilog='''
+Description:
+    dehash.py is a script that tries to dehash whatever log is provided, however
+    it is provided. 'Files' on Hagen-Daas will be consulted if a loghash
+    dictionary isn't specified.
+
+Input File(s):
+    Can be the log to dehash and/or log hash dictionaries to decode the log.
+
+    dehash.py assumes that the hashed log is passed via stdin.
+    If specified in the file list, the hashed log must not have the extension
+    .elf, .pbz, or .json.
+
+    loghash dictionaries can be .json files, .elf files, or bundles (.pbz).
+    Only one dictionary per core may be specified.
+
+Examples:
+    dehash.py pbl-123456.log tintin_fw.elf bt_da14681_main.elf > log.txt
+    dehash.py normal_silk_v4.0-alpha11-20-g6661346.pbz < pbl-12345.log > log.txt
+    gzcat crash_log.gz | dehash.py
+    dehash.py --prf log_from_watch.log
+''')
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument('--fw', action='store_true',
+                       help='Use the fw loghash_dict from your build. Default.')
+    group.add_argument('--prf', action='store_true',
+                       help='Use the prf loghash_dict from your build.')
+    parser.add_argument('-v', action='store_true',
+                        help='Verbose debug to stderr')
+    parser.add_argument('file', nargs='*',
+                        help='Input file(s). See below for more info.')
+    args = parser.parse_args()
+
+    logger = Log(args.v)
+
+    # Make a copy of the file list
+    filelist = list(args.file)
+    # Add the PRF dict to filelist, if appropriate
+    if args.prf:
+        filelist.append(DICT_PRF)
+
+    loghash_dict = {}
+    log = None
+
+    # Examine the file list
+    for f in filelist:
+        if f.endswith('.json') or f.endswith('.elf'):
+            logger.debug('Loading dictionary from %s', f)
+            d = newlogging.get_log_dict_from_file(f)
+            loghash_dict = newlogging.merge_dicts(loghash_dict, d)
+        elif f.endswith('.pbz'):
+            logger.debug('Loading dictionary from %s', f)
+            d = get_dict_from_pbz(f)
+            if not d:
+                raise Exception("Unable to load loghash_dict.json from %s" % f)
+            loghash_dict = newlogging.merge_dicts(loghash_dict, json.loads(d))
+        else:
+            logger.debug('Log file %s', f)
+            if log:
+                raise Exception("More than one log file specified")
+            log = f
+
+    # Now consider the --fw option. Don't fail unless it was explicitly specified
+    if args.fw or (not args.prf and not loghash_dict):
+        logger.debug('Loading dictionary from %s', DICT_FIRMWARE)
+        if os.path.isfile(DICT_FIRMWARE) or args.fw:
+            d = newlogging.get_log_dict_from_file(DICT_FIRMWARE)
+            loghash_dict = newlogging.merge_dicts(loghash_dict, d)
+        else:
+            logger.debug('Ignoring default fw dict -- %s not found', DICT_FIRMWARE)
+
+    # Create the dehasher
+    dehash = logdehash.LogDehash('', monitor_dict_file=False)
+    dehash.load_log_strings_from_dict(loghash_dict)
+
+    # Input file or stdin?
+    infile = open(log) if log else sys.stdin
+
+    # Dehash the log
+    for line in infile:
+        line_dict = dehash.dehash(line)
+        if 'unhashed' in line_dict:
+            dhl = line_dict['formatted_msg']
+        else:
+            dhl = dehash.basic_format_line(line_dict)
+        sys.stdout.write(dhl.strip())
+        sys.stdout.write('\r\n')
+        sys.stdout.flush()
+
+        # If we have a dictionary, continue
+        if loghash_dict:
+            continue
+
+        # No dictionary -- see if we can load one
+        index = dhl.upper().rfind(BUILD_ID_STR)
+        if index == -1:
+            continue
+
+        build_id = dhl[index + len(BUILD_ID_STR):(index + len(BUILD_ID_STR) + HASH_STR_LEN)]
+
+        try:
+            logger.debug('Loading dictionary from Hagen-Daas for ID %s', build_id)
+            d = get_loghash_dict_from_hagen_daas_files(build_id)
+        except (requests.exceptions.ConnectionError,
+                requests.exceptions.HTTPError, AuthException) as error:
+            sys.stderr.write("Could not get build id %s from files. %s\r\n" % (build_id, error))
+            continue
+
+        if d:
+            loghash_dict = json.loads(d)
+            dehash.load_log_strings_from_dict(loghash_dict)
+        else:
+            sys.stderr.write("Could not get build id %s from files.\r\n" % build_id)
+
+    if infile is not sys.stdin:
+        infile.close()
+
+if __name__ == '__main__':
+    main()

tools/log_hashing/logdehash.py

@@ -0,0 +1,222 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# encoding=utf8
+# PBL-31508: This is pretty bad and we want to re-do a lot of this.
+
+import os
+import sys
+import threading
+import time
+from datetime import datetime
+import unicodedata as ud
+
+from pebble.loghashing import newlogging
+from newlogging import get_log_dict_from_file
+
+LOG_DICT_KEY_CORE_ID = 'core_'
+
+COLOR_DICT = {
+               "BLACK":         "\x1b[30m",    "0":  "\x1b[30m",        # Not the most useful
+               "RED":           "\x1b[31m",    "1":  "\x1b[31m",
+               "GREEN":         "\x1b[32m",    "2":  "\x1b[32m",
+               "YELLOW":        "\x1b[33m",    "3":  "\x1b[33m",
+               "BLUE":          "\x1b[34m",    "4":  "\x1b[34m",
+               "MAGENTA":       "\x1b[35m",    "5":  "\x1b[35m",
+               "CYAN":          "\x1b[36m",    "6":  "\x1b[36m",
+               "GREY":          "\x1b[37m",    "7":  "\x1b[37m",
+               "LIGHT_GREY":    "\x1b[1m;30m", "8":  "\x1b[1m;30m",
+               "LIGHT_RED":     "\x1b[1m;31m", "9":  "\x1b[1m;31m",
+               "LIGHT_GREEN":   "\x1b[1m;32m", "10": "\x1b[1m;32m",
+               "LIGHT_YELLOW":  "\x1b[1m;33m", "11": "\x1b[1m;33m",
+               "LIGHT_BLUE":    "\x1b[1m;34m", "12": "\x1b[1m;34m",
+               "LIGHT_MAGENTA": "\x1b[1m;35m", "13": "\x1b[1m;35m",
+               "LIGHT_CYAN":    "\x1b[1m;36m", "14": "\x1b[1m;36m",
+               "WHITE":         "\x1b[1m;37m", "15": "\x1b[1m;37m"}
+COLOR_BOLD_RESET = "\x1b[0m"
+BOLD = "\x1b[1m"
+# Control code to clear the current line
+CLEAR_LINE = "\x1b[2K"
+
+
+class LogDehash(object):
+    """ Dehashing helper with a file update watch thread
+    """
+    def __init__(self, dict_path, justify="small", color=False, bold=-1, print_core=False,
+                 monitor_dict_file=True):
+        self.path = dict_path
+        self.dict_mtime = None
+
+        self.arg_justify = justify
+        self.arg_color = color
+        self.arg_bold = bold
+
+        self.justify_size = 0
+
+        self.load_log_strings()
+        self.print_core = print_core
+        if self.print_core:
+            self.print_core_header()
+
+        self.running = False
+        if monitor_dict_file:
+            self.running = True
+            self.thread = threading.Thread(target=self.run)
+            self.thread.setDaemon(True)
+            self.thread.start()
+
+    def run(self):
+        while self.running:
+            if os.path.lexists(self.path) and (not self.dict_mtime or
+                                               os.path.getmtime(self.path) > self.dict_mtime):
+                # We don't need to worry about thread safety here because the dict is getting
+                # replaced entirely. If anyone has a reference to the old one, it will stay
+                # alive until they're done.
+                self.load_log_strings()
+            time.sleep(5)  # It takes at least this long to flash an update to the board
+
+    def load_log_strings(self):
+        if os.path.lexists(self.path):
+            self.dict_mtime = os.path.getmtime(self.path)
+            self.loghash_dict = get_log_dict_from_file(self.path)
+        else:
+            self.dict_mtime = None
+            self.loghash_dict = None
+        self.update_log_string_metrics()
+
+    def load_log_strings_from_dict(self, dict):
+        self.running = False
+        self.dict_mtime = None
+        self.loghash_dict = dict
+        self.update_log_string_metrics()
+
+    def print_core_header(self):
+        if not self.loghash_dict:
+            return
+
+        print 'Supported Cores:'
+        for key in sorted(self.loghash_dict, key=self.loghash_dict.get):
+            if key.startswith(LOG_DICT_KEY_CORE_ID):
+                print '    {}: {}'.format(key, self.loghash_dict[key])
+
+    def update_log_string_metrics(self):
+        if not self.loghash_dict:
+            self.arg_justify = 0
+            return
+
+        # Handle justification
+        max_basename = 0
+        max_linenum = 0
+        for line_dict in self.loghash_dict.itervalues():
+            if 'file' in line_dict and 'line' in line_dict:
+                max_basename = max(max_basename, len(os.path.basename(line_dict['file'])))
+                max_linenum = max(max_basename, len(os.path.basename(line_dict['line'])))
+        justify_width = max_basename + 1 + max_linenum  # Include the ':'
+
+        if self.arg_justify == 'small':
+            self.justify_size = 0
+        elif self.arg_justify == 'right':
+            self.justify_size = justify_width * -1
+        elif self.arg_justify == 'left':
+            self.justify_size = justify_width
+        else:
+            self.justify_size = int(self.arg_justify)
+
+    def dehash(self, msg):
+        """ Dehashes a logging message.
+        """
+        string = str(msg)
+        if "NL:" in string:  # Newlogging
+            decoded_line = string.decode('utf-8', 'replace')
+            safe_line = ud.normalize('NFKD', decoded_line).encode('utf-8', 'ignore')
+            line_dict = newlogging.dehash_line_unformatted(safe_line, self.loghash_dict)
+            return line_dict
+        else:
+            return {"formatted_msg": string, "unhashed": True}
+
+    def basic_format_line(self, line_dict):
+        output = []
+
+        if 'support' not in line_dict and 're_level' in line_dict:
+            output.append(line_dict['re_level'])
+        if self.print_core and 'core_number' in line_dict:
+            output.append(line_dict['core_number'])
+        if 'task' in line_dict:
+            output.append(line_dict['task'])
+        if 'date' in line_dict:
+            output.append(line_dict['date'])
+        if 'time' in line_dict:
+            output.append(line_dict['time'])
+        elif 'support' not in line_dict:
+            # Use the current time if one isn't provided by the system
+            now = datetime.now()
+            output.append('%02d:%02d:%02d.%03d' % (now.hour, now.minute, now.second,
+                                                   now.microsecond/1000))
+
+        pre_padding = ''
+        post_padding = ''
+
+        if 'file' in line_dict and 'line' in line_dict:
+            filename = os.path.basename(line_dict['file'])
+            file_line = '{}:{}>'.format(filename, line_dict['line'])
+            if self.justify_size < 0:
+                output.append(file_line.rjust(abs(self.justify_size)))
+            else:
+                output.append(file_line.ljust(abs(self.justify_size)))
+
+        output.append(line_dict['formatted_msg'])
+        try:
+            return ' '.join(output)
+        except UnicodeDecodeError:
+            return ''
+
+    def minicom_format_line(self, line_dict):
+        """This routine reformats a line already printed to the console if it was
+        hashed. It does this by clearing the line which was already printed
+        using a control escape command and relies on lines ending in "\r\n".
+        If the line was not hashed in the first place, it simply returns a
+        newline character to print
+
+        """
+        if 'unhashed' in line_dict:
+            return '\n'
+
+        output = []
+        if self.arg_color and 'color' in line_dict:
+            color = line_dict['color']
+            if color in COLOR_DICT:
+                output.append(COLOR_DICT[color])
+        if 'level' in line_dict:
+            if int(line_dict['level']) <= self.arg_bold:
+                output.append(BOLD)
+        output.append(CLEAR_LINE)
+        output.append(self.basic_format_line(line_dict))
+        output.append('\n')
+        output.append(COLOR_BOLD_RESET)
+
+        return ''.join(output)
+
+    def commander_format_line(self, line_dict):
+        output = []
+        if self.arg_color and 'color' in line_dict:
+            color = line_dict['color']
+            if color in COLOR_DICT:
+                output.append(COLOR_DICT[color])
+        if 'level' in line_dict:
+            if int(line_dict['level']) <= self.arg_bold:
+                output.append(BOLD)
+        output.append(self.basic_format_line(line_dict))
+        output.append(COLOR_BOLD_RESET)
+
+        return ''.join(output)

tools/log_hashing/miniterm_co.py

@@ -0,0 +1,166 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# Stuff we're patching or calling into directly
+from serial.tools.miniterm import main
+from serial import Serial
+from serial.urlhandler.protocol_socket import SocketSerial
+from serial.serialutil import SerialException
+
+# Stuff we need
+import os
+import sys
+import threading
+import operator
+import time
+import unicodedata as ud
+import socket
+
+from logdehash import LogDehash
+
+line_buffer = []
+def dehash_read(self, size, plain_read):
+    global line_buffer
+    # Most of the time, we pass through the results of their read command (if rather reconstituted)
+    # At the same time, keep track of the contents of the last line
+    # Once at the end of a line, check if it contains a LH: loghash header
+    #  - If not, continue as usual
+    #  - If it does, dehash the buffered line and use clever tricks to swap it into the terminal
+    #    view in-place (obviously, don't be using this method for raw serial IO or file output)
+    raw_read_data = plain_read(self, size)
+    read_data = []
+    for read_char in raw_read_data:
+        if read_char == "\n":
+            read_line = "".join(line_buffer)
+            line_buffer = []
+            line_dict = dehasher.dehash(read_line)
+            read_data.append(dehasher.minicom_format_line(line_dict))
+        else:
+            line_buffer.append(read_char)
+            read_data.append(read_char)
+    return "".join(read_data)
+
+def socket_serial_read(self, size=1):
+    """
+    Read size bytes from the serial port. If a timeout is set it may
+    return less characters as requested. With no timeout it will block
+    until the requested number of bytes is read.
+    This is a replacement for protocol_socket.SocketSerial.read() that is smarter about
+    handling a closed socket from the remote end. Instead of just immediately returning an
+    empty string, it attempts to reopen the socket when it detects it has closed.
+    """
+    data = bytearray()
+    if self._timeout is not None:
+        timeout = time.time() + self._timeout
+    else:
+        timeout = None
+
+    while len(data) < size and (timeout is None or time.time() < timeout):
+        if not self._isOpen:
+            # If not open, try and re-open
+            try:
+                self.open()
+            except SerialException:
+                # Ignore failure to open and just wait a bit
+                time.sleep(0.1)
+                continue
+
+        try:
+            # Read available data
+            block = self._socket.recv(size - len(data))
+            if block:
+                data.extend(block)
+            else:
+                # no data -> EOF (remote connection closed). If no data at all, loop until
+                # we can reopen the socket
+                self.close()
+                if data:
+                    break
+
+        except socket.timeout:
+            # just need to get out of recv from time to time to check if
+            # still alive
+            continue
+        except socket.error, e:
+            # connection fails -> terminate loop
+            raise SerialException('connection failed (%s)' % e)
+    return bytes(data)
+
+# Insert ourselves in the serial read routine
+#  (could also copy-paste the entire miniterm reader() method in here, which would be meh)
+#  (or patch sys.stdout, which would just suck, because who knows what else that'd break)
+plain_read = Serial.read
+def dehash_serial_read(self, size):
+    return dehash_read(self, size, plain_read)
+
+def dehash_socket_read(self, size):
+    return dehash_read(self, size, socket_serial_read)
+
+try:
+    from pyftdi.serialext.protocol_ftdi import FtdiSerial
+    plain_pyftdi_read = FtdiSerial.read
+    def dehash_pyftdi_serial_read(self, size):
+        return dehash_read(self, size, plain_pyftdi_read)
+    FtdiSerial.read = dehash_pyftdi_serial_read
+except ImportError:
+    pass
+
+
+def yes_no_to_bool(arg):
+    return True if arg == 'yes' else False
+
+
+# Process "arguments"
+arg_justify = "small"
+arg_color = False
+arg_bold = -1
+arg_core = False
+
+dict_path = os.getenv('PBL_CONSOLE_DICT_PATH')
+if not dict_path:
+    dict_path = 'build/src/fw/loghash_dict.json'
+
+arglist = os.getenv("PBL_CONSOLE_ARGS")
+if arglist:
+    for arg in arglist.split(","):
+        if not arg:
+            break
+        key, value = arg.split('=')
+        if key == "--justify":
+            arg_justify = value
+        elif key == "--color":
+            arg_color = yes_no_to_bool(value)
+        elif key == "--bold":
+            arg_bold = int(value)
+        elif key == "--dict":
+            dict_path = value
+        elif key == "--core":
+            arg_core = yes_no_to_bool(value)
+        else:
+            raise Exception("Unknown console argument '{}'. Choices are ({})".
+                            format(key, ['--justify', '--color', '--bold',
+                                         '--dict', '--core']))
+
+dehasher = LogDehash(dict_path, justify=arg_justify,
+                     color=arg_color, bold=arg_bold, print_core=arg_core)
+
+Serial.read = dehash_serial_read
+SocketSerial.read = dehash_socket_read
+
+# Make sure that the target is set
+if sys.argv[1] == 'None':
+    raise Exception("No tty specified. Do you have a device attached?")
+
+# Fire it up as usual
+main()

tools/log_hashing/newlogging.py

@@ -0,0 +1,226 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import re
+import sys
+import json
+import argparse
+
+from elftools.elf.elffile import ELFFile
+from elftools.elf.segments import NoteSegment
+
+
+LOG_STRINGS_SECTION_NAME = ".log_strings"
+BUILD_ID_SECTION_NAME = ".note.gnu.build-id"
+BUILD_ID_NOTE_OWNER_NAME = 'GNU'
+BUILD_ID_NOTE_TYPE_NAME = 'NT_GNU_BUILD_ID'
+
+NEW_LOGGING_VERSION = 'NL0101'
+NEW_LOGGING_HEADER_OFFSET = 0
+
+LOG_LINE_SPLIT_REGEX = r'([^\0]+)\0'
+LOG_LINE_SPLIT_PATTERN = re.compile(r'([^\0]+)\0')   # <anything but '\0'>  followed by '\0'
+
+LOG_LINE_KEY_ALL_REGEX = '(?P{}.*)'                    # matches anything (with match group name {})
+LOG_LINE_KEY_NO_EMBEDDED_COLON_REGEX = '(?P{}[^:]*?)'  # matches anything not containing a colon
+
+LOG_DICT_KEY_BUILD_ID_LEGACY = 'build_id'
+LOG_DICT_KEY_BUILD_ID = 'build_id_core_'
+LOG_DICT_KEY_VERSION = 'new_logging_version'
+LOG_DICT_KEY_CORE_ID = 'core_'
+
+LOG_CORE_ID_OFFSET_SHIFT = 30
+
+
+class LogDict:
+    def __init__(self):
+        self.log_dict = {}
+        key_list = []
+        core_id = 0
+        core_id_offset = 0
+        core_name = ''
+        log_line_regex = None
+
+    def parse_header_from_section(self, section):
+        # Split the header string from the log strings, which follow
+        header_string = section[:section.find('\0')]
+        # Split the header string into parts, comma delimited
+        for line in header_string.split(','):
+            tag, key = line.split('=')
+            if tag.startswith('NL'):
+                self.log_dict[LOG_DICT_KEY_VERSION] = tag
+                self.key_list = key.split(':')
+            elif tag.startswith('CORE_ID'):
+                self.core_id = key
+            elif tag.startswith('CORE_NAME'):
+                self.core_name = key
+            else:
+                raise Exception("Unknown header tag '{}'".format(line))
+
+        if self.log_dict[LOG_DICT_KEY_VERSION] != NEW_LOGGING_VERSION:
+            # Worthy of an exception! Something bad has happened with the tools configuration.
+            raise Exception("Expected log strings version {} not {}".format(NEW_LOGGING_VERSION,
+                                                                            version))
+
+    def log_line_regex_from_key_list(self):
+        regex = []
+        for key in self.key_list:
+            if key == '<msg>':
+                regex.append(LOG_LINE_KEY_ALL_REGEX.format(key))  # Allow embedded colons
+            else:
+                regex.append(LOG_LINE_KEY_NO_EMBEDDED_COLON_REGEX.format(key))  # No embedded colons
+
+        self.log_line_regex = re.compile(":".join(regex))
+
+    def set_section_and_build_id(self, section, build_id):
+        # Parse the header
+        self.parse_header_from_section(section)
+
+        # Now split the log line into a key dict using the generated regex
+        self.log_line_regex_from_key_list()
+
+        # Set the build id. If core 0, use the legacy build ID.
+        # Otherwise, use the new style build ID -- append the core ID.
+        if self.core_id == '0':
+            self.log_dict[LOG_DICT_KEY_BUILD_ID_LEGACY] = build_id
+        else:
+            self.log_dict[LOG_DICT_KEY_BUILD_ID + self.core_id] = build_id
+
+        # Set the core ID
+        self.log_dict[LOG_DICT_KEY_CORE_ID + self.core_id] = self.core_name
+
+        # Calculate the core/id offset
+        self.core_id_offset = int(self.core_id) << LOG_CORE_ID_OFFSET_SHIFT
+
+        # Create a log dictionary of line dictionaries.
+        # { 'offset', { '<file>': <filename>, '<line>': <line>, '<level>': <level>, '<msg>': <msg>,
+        #             'color': <color> }}
+        # Stupidly, offset is now a string because JSON sucks
+        # 'offset' is a decimal string.
+
+        # First, split the section by '\0' characters, keeping track of the start.
+        # There may be padding, so ignore any empty strings.
+        for line in LOG_LINE_SPLIT_PATTERN.finditer(section):
+            # Skip the header line
+            if line.start() == NEW_LOGGING_HEADER_OFFSET:
+                continue
+
+            tags = self.log_line_regex.match(line.group(1))
+
+            # Add the core offset to the 'offset' parameter
+            self.log_dict[str(line.start() + self.core_id_offset)] = tags.groupdict()
+
+    def get_log_dict(self):
+        return self.log_dict
+
+
+def get_elf_section(filename, section_name):
+    with open(filename, 'rb') as file:
+        section = ELFFile(file).get_section_by_name(section_name)
+        return section.data() if section is not None else None
+
+
+def get_elf_build_id(filename):
+    with open(filename, 'rb') as file:
+        for segment in ELFFile(file).iter_segments():
+            if isinstance(segment, NoteSegment):
+                for note in segment.iter_notes():
+                    if note['n_name'] == BUILD_ID_NOTE_OWNER_NAME and note['n_type'] == \
+                                                                        BUILD_ID_NOTE_TYPE_NAME:
+                        return note['n_desc']
+    return ''
+
+
+""" Merge two loghash dictionaries and return the union.
+    The first dictionary may be empty """
+
+
+def merge_dicts(dict1, dict2):
+    # Check to make sure that the versions exist and are identical
+    if len(dict1) and LOG_DICT_KEY_VERSION not in dict1:
+        raise Exception('First log_dict does not contain version info')
+
+        if LOG_DICT_KEY_VERSION not in dict2:
+            raise Exception('Second log_dict does not contain version info')
+
+        if dict1[LOG_DICT_KEY_VERSION] != dict2[LOG_DICT_KEY_VERSION]:
+            raise Exception('log dicts have different versions! {} != {}'.format(
+                            dict1[LOG_DICT_KEY_VERSION], dict2[LOG_DICT_KEY_VERSION]))
+
+    # Check to make sure that both have core IDs and that they're different.
+    core_list_dict1 = []
+    for key in dict1:
+        if key.startswith(LOG_DICT_KEY_CORE_ID):
+            core_list_dict1.append(key)
+    core_list_dict2 = []
+    for key in dict2:
+        if key.startswith(LOG_DICT_KEY_CORE_ID):
+            core_list_dict2.append(key)
+
+    if len(dict1) and len(core_list_dict1) == 0:
+        raise Exception('First log_dict does not specify a core_id')
+    if len(core_list_dict2) == 0:
+        raise Exception('Second log_dict does not specify a core_id')
+
+    intersection = set(core_list_dict1).intersection(core_list_dict2)
+    if len(intersection) != 0:
+        raise Exception('Both log_dicts specify the following cores: {}'.format(list(intersection)))
+
+    # Merge the dictionaries. Don't bother confirming there are no log message conflicts.
+    merged_dict = dict1.copy()
+    merged_dict.update(dict2)
+
+    return merged_dict
+
+
+""" ----------------------------- External API -------------------------------- """
+
+
+""" Returns log dict (including build_id and new_logging_version keys) from specified
+    filename. Accepts either tintin_fw.elf or loghash_dict.json """
+
+
+def get_log_dict_from_file(filename):
+
+    if filename.endswith('.json'):
+        with open(filename, 'rb') as file:
+            return json.load(file)
+
+    log_strings_section = get_elf_section(filename, LOG_STRINGS_SECTION_NAME)
+    build_id = get_elf_build_id(filename)
+
+    ld = LogDict()
+    ld.set_section_and_build_id(log_strings_section, build_id)
+
+    return ld.get_log_dict()
+
+
+""" Merge the loghash_dict.json files named in 'merge_list' to node 'out_file' """
+
+
+def merge_loghash_dict_json_files(out_file, merge_list):
+    out_dict = {}
+
+    for json_file in merge_list:
+        with open(json_file.abspath(), 'r') as json_file:
+            in_dict = json.load(json_file)
+            out_dict = merge_dicts(out_dict, in_dict)
+
+    if LOG_DICT_KEY_BUILD_ID_LEGACY not in out_dict:
+        raise Exception('build_id missing from loghash_dict.json')
+
+    with open(out_file.abspath(), 'w') as json_file:
+        json.dump(out_dict, json_file, indent=2, sort_keys=True)
+
+