Mirrors/pebble
1
0
Fork 0
mirror of https://github.com/google/pebble.git synced 2025-05-21 10:55:04 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Import of the watch repository from Pebble

Browse source
This commit is contained in:
Matthieu Jeanson 2024-12-12 16:43:03 -08:00 • committed by Katharine Berry
commit 3b92768480
10334 changed files with 2564465 additions and 0 deletions
Download patch file Download diff file Expand all files Collapse all files

15
python_libs/pulse2/pebble/__init__.py Normal file
View file

@ -0,0 +1,15 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
__import__('pkg_resources').declare_namespace(__name__)

24
python_libs/pulse2/pebble/pulse2/__init__.py Normal file
View file

@ -0,0 +1,24 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
from . import link, transports
# Public aliases for the classes that users will interact with directly.
from .link import Interface
link.Link.register_transport(
'best-effort', transports.BestEffortApplicationTransport)
link.Link.register_transport('reliable', transports.ReliableTransport)

37
python_libs/pulse2/pebble/pulse2/exceptions.py Normal file
View file

@ -0,0 +1,37 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
class PulseException(Exception):
pass
class TTYAutodetectionUnavailable(PulseException):
pass
class ReceiveQueueEmpty(PulseException):
pass
class TransportNotReady(PulseException):
pass
class SocketClosed(PulseException):
pass
class AlreadyInProgressError(PulseException):
'''Another operation is already in progress.
'''

148
python_libs/pulse2/pebble/pulse2/framing.py Normal file
View file

@ -0,0 +1,148 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
'''
PULSEv2 Framing
This module handles encoding and decoding of datagrams in PULSEv2 frames: flag
delimiters, transparency encoding and Frame Check Sequence. The content of the
datagrams themselves are not examined or parsed.
'''
from __future__ import absolute_import
import binascii
import struct
try:
import queue
except ImportError: # Py2
import Queue as queue
from cobs import cobs
FLAG = 0x55
CRC32_RESIDUE = binascii.crc32(b'\0' * 4)
class FramingException(Exception):
pass
class DecodeError(FramingException):
pass
class CorruptFrame(FramingException):
pass
class FrameSplitter(object):
'''Takes a byte stream and partitions it into frames.
Empty frames (two consecutive flag bytes) are silently discarded.
No transparency conversion is applied to the contents of the frames.
FrameSplitter objects support iteration for retrieving split frames.
>>> splitter = FrameSplitter()
>>> splitter.write(b'\x55foo\x55bar\x55')
>>> list(splitter)
[b'foo', b'bar']
'''
def __init__(self, max_frame_length=0):
self.frames = queue.Queue()
self.input_buffer = bytearray()
self.max_frame_length = max_frame_length
self.waiting_for_sync = True
def write(self, data):
'''Write bytes into the splitter for processing.
'''
for char in bytearray(data):
if self.waiting_for_sync:
if char == FLAG:
self.waiting_for_sync = False
else:
if char == FLAG:
if self.input_buffer:
self.frames.put_nowait(bytes(self.input_buffer))
self.input_buffer = bytearray()
else:
if (not self.max_frame_length or
len(self.input_buffer) < self.max_frame_length):
self.input_buffer.append(char)
else:
self.input_buffer = bytearray()
self.waiting_for_sync = True
def __iter__(self):
while True:
try:
yield self.frames.get_nowait()
except queue.Empty:
return
def decode_transparency(frame_bytes):
'''Decode the transparency encoding applied to a PULSEv2 frame.
Returns the decoded frame, or raises `DecodeError`.
'''
frame_bytes = bytearray(frame_bytes)
if FLAG in frame_bytes:
raise DecodeError("flag byte in encoded frame")
try:
return cobs.decode(bytes(frame_bytes.replace(b'\0', bytearray([FLAG]))))
except cobs.DecodeError as e:
raise DecodeError(str(e))
def strip_fcs(frame_bytes):
'''Validates the FCS in a PULSEv2 frame.
The frame is returned with the FCS removed if the FCS check passes.
A `CorruptFrame` exception is raised if the FCS check fails.
The frame must not be transparency-encoded.
'''
if len(frame_bytes) <= 4:
raise CorruptFrame('frame too short')
if binascii.crc32(frame_bytes) != CRC32_RESIDUE:
raise CorruptFrame('FCS check failure')
return frame_bytes[:-4]
def decode_frame(frame_bytes):
'''Decode and validate a PULSEv2-encoded frame.
Returns the datagram extracted from the frame, or raises a
`FramingException` or subclass if there was an error decoding the frame.
'''
return strip_fcs(decode_transparency(frame_bytes))
def encode_frame(datagram):
'''Encode a datagram in a PULSEv2 frame.
'''
datagram = bytearray(datagram)
fcs = binascii.crc32(datagram) & 0xffffffff
fcs_bytes = struct.pack('<I', fcs)
datagram.extend(fcs_bytes)
flag = bytearray([FLAG])
frame = cobs.encode(bytes(datagram)).replace(flag, b'\0')
return flag + frame + flag

314
python_libs/pulse2/pebble/pulse2/link.py Normal file
View file

@ -0,0 +1,314 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
import logging
import threading
import serial
from . import exceptions, framing, ppp, transports
from . import logging as pulse2_logging
from . import pcap_file
try:
import pyftdi.serialext
except ImportError:
pass
logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())
DBGSERIAL_PORT_SETTINGS = dict(baudrate=1000000, timeout=0.1,
interCharTimeout=0.0001)
def get_dbgserial_tty():
# Local import so that we only depend on this package if we're attempting
# to autodetect the TTY. This package isn't always available (e.g., MFG),
# so we don't want it to be required.
try:
import pebble_tty
return pebble_tty.find_dbgserial_tty()
except ImportError:
raise exceptions.TTYAutodetectionUnavailable
class Interface(object):
'''The PULSEv2 lower data-link layer.
An Interface object is roughly analogous to a network interface,
somewhat like an Ethernet port. It provides connectionless service
with PULSEv2 framing, which upper layers build upon to provide
connection-oriented service.
An Interface is bound to an I/O stream, such as a Serial port, and
remains open until either the Interface is explicitly closed or the
underlying I/O stream is closed from underneath it.
'''
def __init__(self, iostream, capture_stream=None):
self.logger = pulse2_logging.TaggedAdapter(
logger, {'tag': type(self).__name__})
self.iostream = iostream
self.closed = False
self.close_lock = threading.RLock()
self.default_packet_handler_cb = None
self.sockets = {}
self.pcap = None
if capture_stream:
self.pcap = pcap_file.PcapWriter(
capture_stream, pcap_file.LINKTYPE_PPP_WITH_DIR)
self.receive_thread = threading.Thread(target=self.receive_loop)
self.receive_thread.daemon = True
self.receive_thread.start()
self.simplex_transport = transports.SimplexTransport(self)
self._link = None
self.link_available = threading.Event()
self.lcp = ppp.LinkControlProtocol(self)
self.lcp.on_link_up = self.on_link_up
self.lcp.on_link_down = self.on_link_down
self.lcp.up()
self.lcp.open()
@classmethod
def open_dbgserial(cls, url=None, capture_stream=None):
if url is None:
url = get_dbgserial_tty()
elif url == 'qemu':
url = 'socket://localhost:12345'
ser = serial.serial_for_url(url, **DBGSERIAL_PORT_SETTINGS)
if url.startswith('socket://'):
# interCharTimeout doesn't apply to sockets, so shrink the receive
# timeout to compensate.
ser.timeout = 0.5
ser._socket.settimeout(0.5)
return cls(ser, capture_stream)
def connect(self, protocol):
'''Open a link-layer socket for sending and receiving packets
of a specific protocol number.
'''
if protocol in self.sockets and not self.sockets[protocol].closed:
raise ValueError('A socket is already bound '
'to protocol 0x%04x' % protocol)
self.sockets[protocol] = socket = InterfaceSocket(self, protocol)
return socket
def unregister_socket(self, protocol):
'''Used by InterfaceSocket objets to unregister themselves when
closing.
'''
try:
del self.sockets[protocol]
except KeyError:
pass
def receive_loop(self):
splitter = framing.FrameSplitter()
while True:
if self.closed:
self.logger.info('Interface closed; receive loop exiting')
break
try:
splitter.write(self.iostream.read(1))
except IOError:
if self.closed:
self.logger.info('Interface closed; receive loop exiting')
else:
self.logger.exception('Unexpected error while reading '
'from iostream')
self._down()
break
for frame in splitter:
try:
datagram = framing.decode_frame(frame)
if self.pcap:
# Prepend pseudo-header meaning "received by this host"
self.pcap.write_packet(b'\0' + datagram)
protocol, information = ppp.unencapsulate(datagram)
if protocol in self.sockets:
self.sockets[protocol].handle_packet(information)
else:
# TODO LCP Protocol-Reject
self.logger.info('Protocol-reject: %04X', protocol)
except (framing.DecodeError, framing.CorruptFrame):
pass
def send_packet(self, protocol, packet):
if self.closed:
raise ValueError('I/O operation on closed interface')
datagram = ppp.encapsulate(protocol, packet)
if self.pcap:
# Prepend pseudo-header meaning "sent by this host"
self.pcap.write_packet(b'\x01' + datagram)
self.iostream.write(framing.encode_frame(datagram))
def close_all_sockets(self):
# Iterating over a copy of sockets since socket.close() can call
# unregister_socket, which modifies the socket dict. Modifying
# a dict during iteration is not allowed, so the iteration is
# completed (by making the copy) before modification can begin.
for socket in list(self.sockets.values()):
socket.close()
def close(self):
with self.close_lock:
if self.closed:
return
self.lcp.shutdown()
self.close_all_sockets()
self._down()
if self.pcap:
self.pcap.close()
def _down(self):
'''The lower layer (iostream) is down. Bring down the interface.
'''
with self.close_lock:
self.closed = True
self.close_all_sockets()
self.lcp.down()
self.simplex_transport.down()
self.iostream.close()
def on_link_up(self):
# FIXME PBL-34320 proper MTU/MRU support
self._link = Link(self, mtu=1500)
# Test whether the link is ready to carry traffic
self.lcp.ping(self._ping_done)
def _ping_done(self, ping_check_succeeded):
if ping_check_succeeded:
self.link_available.set()
else:
self.lcp.restart()
def on_link_down(self):
self.link_available.clear()
self._link.down()
self._link = None
def get_link(self, timeout=60.0):
'''Get the opened Link object for this interface.
This function will block waiting for the Link to be available.
It will return `None` if the timeout expires before the link
is available.
'''
if self.closed:
raise ValueError('No link available on closed interface')
if self.link_available.wait(timeout):
assert self._link is not None
return self._link
class InterfaceSocket(object):
'''A socket for sending and receiving link-layer packets over a
PULSE2 interface.
Callbacks can be registered on the socket by assigning callables to
the appropriate attributes on the socket object. Callbacks can be
unregistered by setting the attributes back to `None`.
Available callbacks:
- `on_packet(information)`
- `on_close()`
'''
on_packet = None
on_close = None
def __init__(self, interface, protocol):
self.interface = interface
self.protocol = protocol
self.closed = False
def __enter__(self):
return self
def __exit__(self, type, value, traceback):
self.close()
def send(self, information):
if self.closed:
raise exceptions.SocketClosed('I/O operation on closed socket')
self.interface.send_packet(self.protocol, information)
def handle_packet(self, information):
if self.on_packet and not self.closed:
self.on_packet(information)
def close(self):
if self.closed:
return
self.closed = True
if self.on_close:
self.on_close()
self.interface.unregister_socket(self.protocol)
self.on_packet = None
self.on_close = None
class Link(object):
'''The connectionful portion of a PULSE2 interface.
'''
TRANSPORTS = {}
on_close = None
@classmethod
def register_transport(cls, name, factory):
'''Register a PULSE transport.
'''
if name in cls.TRANSPORTS:
raise ValueError('transport name %r is already registered '
'with %r' % (name, cls.TRANSPORTS[name]))
cls.TRANSPORTS[name] = factory
def __init__(self, interface, mtu):
self.logger = pulse2_logging.TaggedAdapter(
logger, {'tag': type(self).__name__})
self.interface = interface
self.closed = False
self.mtu = mtu
self.transports = {}
for name, factory in self.TRANSPORTS.iteritems():
transport = factory(interface, mtu)
self.transports[name] = transport
def open_socket(self, transport, port, timeout=30.0):
if self.closed:
raise ValueError('Cannot open socket on closed Link')
if transport not in self.transports:
raise KeyError('Unknown transport %r' % transport)
return self.transports[transport].open_socket(port, timeout)
def down(self):
self.closed = True
if self.on_close:
self.on_close()
for transport in self.transports.itervalues():
transport.down()

31
python_libs/pulse2/pebble/pulse2/logging.py Normal file
View file

@ -0,0 +1,31 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
import logging
class TaggedAdapter(logging.LoggerAdapter):
'''Annotates all log messages with a "[tag]" prefix.
The value of the tag is specified in the dict argument passed into
the adapter's constructor.
>>> logger = logging.getLogger(__name__)
>>> adapter = TaggedAdapter(logger, {'tag': 'tag value'})
'''
def process(self, msg, kwargs):
return '[%s] %s' % (self.extra['tag'], msg), kwargs

68
python_libs/pulse2/pebble/pulse2/pcap_file.py Normal file
View file

@ -0,0 +1,68 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
'''Writer for Libpcap capture files
https://wiki.wireshark.org/Development/LibpcapFileFormat
'''
from __future__ import absolute_import
import struct
import threading
import time
LINKTYPE_PPP_WITH_DIR = 204
class PcapWriter(object):
def __init__(self, outfile, linktype):
self.lock = threading.Lock()
self.outfile = outfile
self._write_pcap_header(linktype)
def close(self):
with self.lock:
self.outfile.close()
def __enter__(self):
return self
def __exit__(self, *args):
self.close()
def _write_pcap_header(self, linktype):
header = struct.pack('!IHHiIII',
0xa1b2c3d4, # guint32 magic_number
2, # guint16 version_major
4, # guint16 version_minor
0, # guint32 thiszone
0, # guint32 sigfigs (unused)
65535, # guint32 snaplen
linktype) # guint32 network
self.outfile.write(header)
def write_packet(self, data, timestamp=None, orig_len=None):
assert len(data) <= 65535
if timestamp is None:
timestamp = time.time()
if orig_len is None:
orig_len = len(data)
ts_seconds = int(timestamp)
ts_usec = int((timestamp - ts_seconds) * 1000000)
header = struct.pack('!IIII', ts_seconds, ts_usec, len(data), orig_len)
with self.lock:
self.outfile.write(header + data)

201
python_libs/pulse2/pebble/pulse2/pcmp.py Normal file
View file

@ -0,0 +1,201 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
'''PULSE Control Message Protocol
'''
from __future__ import absolute_import
import codecs
import collections
import enum
import logging
import struct
import threading
from . import exceptions
from . import logging as pulse2_logging
logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())
class ParseError(exceptions.PulseException):
pass
@enum.unique
class PCMPCode(enum.Enum):
Echo_Request = 1
Echo_Reply = 2
Discard_Request = 3
Port_Closed = 129
Unknown_Code = 130
class PCMPPacket(collections.namedtuple('PCMPPacket', 'code information')):
__slots__ = ()
@classmethod
def parse(cls, packet):
packet = bytes(packet)
if len(packet) < 1:
raise ParseError('packet too short')
return cls(code=struct.unpack('B', packet[0])[0],
information=packet[1:])
@staticmethod
def build(code, information):
return struct.pack('B', code) + bytes(information)
class PulseControlMessageProtocol(object):
'''This protocol is unique in that it is logically part of the
transport but is layered on top of the transport over the wire.
To keep from needing to create a new thread just for reading from
the socket, the implementation acts both like a socket and protocol
all in one.
'''
PORT = 0x0001
on_port_closed = None
@classmethod
def bind(cls, transport):
return transport.open_socket(cls.PORT, factory=cls)
def __init__(self, transport, port):
assert port == self.PORT
self.logger = pulse2_logging.TaggedAdapter(
logger, {'tag': 'PCMP(%s)' % (type(transport).__name__)})
self.transport = transport
self.closed = False
self.ping_lock = threading.RLock()
self.ping_cb = None
self.ping_attempts_remaining = 0
self.ping_timer = None
def close(self):
if self.closed:
return
with self.ping_lock:
self.ping_cb = None
if self.ping_timer:
self.ping_timer.cancel()
self.closed = True
self.transport.unregister_socket(self.PORT)
def send_unknown_code(self, bad_code):
self.transport.send(self.PORT, PCMPPacket.build(
PCMPCode.Unknown_Code.value, struct.pack('B', bad_code)))
def send_echo_request(self, data):
self.transport.send(self.PORT, PCMPPacket.build(
PCMPCode.Echo_Request.value, data))
def send_echo_reply(self, data):
self.transport.send(self.PORT, PCMPPacket.build(
PCMPCode.Echo_Reply.value, data))
def on_receive(self, raw_packet):
try:
packet = PCMPPacket.parse(raw_packet)
except ParseError:
self.logger.exception('Received malformed packet')
return
try:
code = PCMPCode(packet.code)
except ValueError:
self.logger.error('Received packet with unknown code %d',
packet.code)
self.send_unknown_code(packet.code)
return
if code == PCMPCode.Discard_Request:
pass
elif code == PCMPCode.Echo_Request:
self.send_echo_reply(packet.information)
elif code == PCMPCode.Echo_Reply:
with self.ping_lock:
if self.ping_cb:
self.ping_timer.cancel()
self.ping_cb(True)
self.ping_cb = None
self.logger.debug('Echo-Reply: %s',
codecs.encode(packet.information, 'hex'))
elif code == PCMPCode.Port_Closed:
if len(packet.information) == 2:
if self.on_port_closed:
closed_port, = struct.unpack('!H', packet.information)
self.on_port_closed(closed_port)
else:
self.logger.error(
'Remote peer sent malformed Port-Closed packet: %s',
codecs.encode(packet.information, 'hex'))
elif code == PCMPCode.Unknown_Code:
if len(packet.information) == 1:
self.logger.error('Remote peer sent Unknown-Code(%d) packet',
struct.unpack('B', packet.information)[0])
else:
self.logger.error(
'Remote peer sent malformed Unknown-Code packet: %s',
codecs.encode(packet.information, 'hex'))
else:
assert False, 'Known code not handled'
def ping(self, result_cb, attempts=3, timeout=1.0):
'''Test the link quality by sending Echo-Request packets and
listening for Echo-Reply packets from the remote peer.
The ping is performed asynchronously. The `result_cb` callable
will be called when the ping completes. It will be called with
a single positional argument: a truthy value if the remote peer
responded to the ping, or a falsy value if all ping attempts
timed out.
'''
if attempts < 1:
raise ValueError('attempts must be positive')
if timeout <= 0:
raise ValueError('timeout must be positive')
with self.ping_lock:
if self.ping_cb:
raise exceptions.AlreadyInProgressError(
'another ping is currently in progress')
self.ping_cb = result_cb
self.ping_attempts_remaining = attempts - 1
self.ping_timeout = timeout
self.send_echo_request(b'')
self.ping_timer = threading.Timer(timeout,
self._ping_timer_expired)
self.ping_timer.daemon = True
self.ping_timer.start()
def _ping_timer_expired(self):
with self.ping_lock:
if not self.ping_cb:
# The Echo-Reply packet must have won the race
return
if self.ping_attempts_remaining:
self.ping_attempts_remaining -= 1
self.send_echo_request(b'')
self.ping_timer = threading.Timer(self.ping_timeout,
self._ping_timer_expired)
self.ping_timer.daemon = True
self.ping_timer.start()
else:
self.ping_cb(False)
self.ping_cb = None

1094
python_libs/pulse2/pebble/pulse2/ppp.py Normal file
View file

File diff suppressed because it is too large Load diff

68
python_libs/pulse2/pebble/pulse2/stats.py Normal file
View file

@ -0,0 +1,68 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import, division
import math
class OnlineStatistics(object):
'''Calculates various statistical properties of a data series
iteratively, without keeping the data items in memory.
Available statistics:
- Count
- Min
- Max
- Mean
- Variance
- Standard Deviation
The variance calculation algorithm is taken from
https://en.wikipedia.org/w/index.php?title=Algorithms_for_calculating_variance&oldid=715886413#Online_algorithm
'''
def __init__(self):
self.count = 0
self.min = float('nan')
self.max = float('nan')
self.mean = 0.0
self.M2 = 0.0
def update(self, datum):
self.count += 1
if self.count == 1:
self.min = datum
self.max = datum
else:
self.min = min(self.min, datum)
self.max = max(self.max, datum)
delta = datum - self.mean
self.mean += delta / self.count
self.M2 += delta * (datum - self.mean)
@property
def variance(self):
if self.count < 2:
return float('nan')
return self.M2 / (self.count - 1)
@property
def stddev(self):
return math.sqrt(self.variance)
def __str__(self):
return 'min/avg/max/stddev = {:.03f}/{:.03f}/{:.03f}/{:.03f}'.format(
self.min, self.mean, self.max, self.stddev)

628
python_libs/pulse2/pebble/pulse2/transports.py Normal file
View file

@ -0,0 +1,628 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import absolute_import
import logging
import threading
import time
try:
import queue
except ImportError:
import Queue as queue
import construct
from . import exceptions
from . import logging as pulse2_logging
from . import pcmp
from . import ppp
from . import stats
logger = logging.getLogger(__name__)
logger.addHandler(logging.NullHandler())
class Socket(object):
'''A socket for sending and receiving packets over a single port
of a PULSE2 transport.
'''
def __init__(self, transport, port):
self.transport = transport
self.port = port
self.closed = False
self.receive_queue = queue.Queue()
def on_receive(self, packet):
self.receive_queue.put((True, packet))
def receive(self, block=True, timeout=None):
if self.closed:
raise exceptions.SocketClosed('I/O operation on closed socket')
try:
info_good, info = self.receive_queue.get(block, timeout)
if not info_good:
assert self.closed
raise exceptions.SocketClosed('Socket closed during receive')
return info
except queue.Empty:
raise exceptions.ReceiveQueueEmpty
def send(self, information):
if self.closed:
raise exceptions.SocketClosed('I/O operation on closed socket')
self.transport.send(self.port, information)
def close(self):
if self.closed:
return
self.closed = True
self.transport.unregister_socket(self.port)
# Wake up the thread blocking on a receive (if any) so that it
# can abort the receive quickly.
self.receive_queue.put((False, None))
@property
def mtu(self):
return self.transport.mtu
class TransportControlProtocol(ppp.ControlProtocol):
def __init__(self, interface, transport, ncp_protocol, display_name=None):
ppp.ControlProtocol.__init__(self, display_name)
self.interface = interface
self.ncp_protocol = ncp_protocol
self.transport = transport
def up(self):
ppp.ControlProtocol.up(self, self.interface.connect(self.ncp_protocol))
def this_layer_up(self, *args):
self.transport.this_layer_up()
def this_layer_down(self, *args):
self.transport.this_layer_down()
BestEffortPacket = construct.Struct('BestEffortPacket', # noqa
construct.UBInt16('port'),
construct.UBInt16('length'),
construct.Field('information', lambda ctx: ctx.length - 4),
ppp.OptionalGreedyString('padding'),
)
class BestEffortTransportBase(object):
def __init__(self, interface, link_mtu):
self.logger = pulse2_logging.TaggedAdapter(
logger, {'tag': type(self).__name__})
self.sockets = {}
self.closed = False
self._mtu = link_mtu - 4
self.link_socket = interface.connect(self.PROTOCOL_NUMBER)
self.link_socket.on_packet = self.packet_received
def send(self, port, information):
if len(information) > self.mtu:
raise ValueError('Packet length (%d) exceeds transport MTU (%d)' % (
len(information), self.mtu))
packet = BestEffortPacket.build(construct.Container(
port=port, length=len(information)+4,
information=information, padding=b''))
self.link_socket.send(packet)
def packet_received(self, packet):
if self.closed:
self.logger.warning('Received packet on closed transport')
return
try:
fields = BestEffortPacket.parse(packet)
except (construct.ConstructError, ValueError):
self.logger.exception('Received malformed packet')
return
if len(fields.information) + 4 != fields.length:
self.logger.error('Received truncated or corrupt packet '
'(expected %d, got %d data bytes)',
fields.length-4, len(fields.information))
return
if fields.port in self.sockets:
self.sockets[fields.port].on_receive(fields.information)
else:
self.logger.warning('Received packet for unopened port %04X',
fields.port)
def open_socket(self, port, factory=Socket):
if self.closed:
raise ValueError('Cannot open socket on closed transport')
if port in self.sockets and not self.sockets[port].closed:
raise KeyError('Another socket is already opened '
'on port 0x%04x' % port)
socket = factory(self, port)
self.sockets[port] = socket
return socket
def unregister_socket(self, port):
del self.sockets[port]
def down(self):
'''Called by the Link when the link layer goes down.
This closes the Transport object. Once closed, the Transport
cannot be reopened.
'''
self.closed = True
self.close_all_sockets()
self.link_socket.close()
def close_all_sockets(self):
# A socket could try to unregister itself when closing, which
# would modify the sockets dict. Make a copy of the sockets
# collection before closing them so that we are not iterating
# over the dict when it could get modified.
for socket in list(self.sockets.values()):
socket.close()
self.sockets = {}
@property
def mtu(self):
return self._mtu
class BestEffortApplicationTransport(BestEffortTransportBase):
NCP_PROTOCOL_NUMBER = 0xBA29
PROTOCOL_NUMBER = 0x3A29
def __init__(self, interface, link_mtu):
BestEffortTransportBase.__init__(self, interface=interface,
link_mtu=link_mtu)
self.opened = threading.Event()
self.ncp = TransportControlProtocol(
interface=interface, transport=self,
ncp_protocol=self.NCP_PROTOCOL_NUMBER,
display_name='BestEffortControlProtocol')
self.ncp.up()
self.ncp.open()
def this_layer_up(self):
# We can't let PCMP bind itself using the public open_socket
# method as the method will block until self.opened is set, but
# it won't be set until we use PCMP Echo to test that the
# transport is ready to carry traffic. So we must manually bind
# the port without waiting.
self.pcmp = pcmp.PulseControlMessageProtocol(
self, pcmp.PulseControlMessageProtocol.PORT)
self.sockets[pcmp.PulseControlMessageProtocol.PORT] = self.pcmp
self.pcmp.on_port_closed = self.on_port_closed
self.pcmp.ping(self._ping_done)
def _ping_done(self, ping_check_succeeded):
# Don't need to do anything in the success case as receiving
# any packet is enough to set the transport as Opened.
if not ping_check_succeeded:
self.logger.warning('Ping check failed. Restarting transport.')
self.ncp.restart()
def this_layer_down(self):
self.opened.clear()
self.close_all_sockets()
def send(self, *args, **kwargs):
if self.closed:
raise exceptions.TransportNotReady(
'I/O operation on closed transport')
if not self.ncp.is_Opened():
raise exceptions.TransportNotReady(
'I/O operation before transport is opened')
BestEffortTransportBase.send(self, *args, **kwargs)
def packet_received(self, packet):
if self.ncp.is_Opened():
self.opened.set()
BestEffortTransportBase.packet_received(self, packet)
else:
self.logger.warning('Received packet before the transport is open. '
'Discarding.')
def open_socket(self, port, timeout=30.0, factory=Socket):
if not self.opened.wait(timeout):
return None
return BestEffortTransportBase.open_socket(self, port, factory)
def down(self):
self.ncp.down()
BestEffortTransportBase.down(self)
def on_port_closed(self, closed_port):
self.logger.info('Remote peer says port 0x%04X is closed; '
'closing socket', closed_port)
try:
self.sockets[closed_port].close()
except KeyError:
self.logger.exception('No socket is open on port 0x%04X!',
closed_port)
class SimplexTransport(BestEffortTransportBase):
PROTOCOL_NUMBER = 0x5021
def __init__(self, interface):
BestEffortTransportBase.__init__(self, interface=interface, link_mtu=0)
def send(self, *args, **kwargs):
raise NotImplementedError
@property
def mtu(self):
return 0
ReliableInfoPacket = construct.Struct('ReliableInfoPacket', # noqa
# BitStructs are parsed MSBit-first
construct.EmbeddedBitStruct(
construct.BitField('sequence_number', 7), # N(S) in LAPB
construct.Const(construct.Bit('discriminator'), 0),
construct.BitField('ack_number', 7), # N(R) in LAPB
construct.Flag('poll'),
),
construct.UBInt16('port'),
construct.UBInt16('length'),
construct.Field('information', lambda ctx: ctx.length - 6),
ppp.OptionalGreedyString('padding'),
)
ReliableSupervisoryPacket = construct.BitStruct(
'ReliableSupervisoryPacket',
construct.Const(construct.Nibble('reserved'), 0b0000),
construct.Enum(construct.BitField('kind', 2), # noqa
RR=0b00,
RNR=0b01,
REJ=0b10,
),
construct.Const(construct.BitField('discriminator', 2), 0b01),
construct.BitField('ack_number', 7), # N(R) in LAPB
construct.Flag('poll'),
construct.Alias('final', 'poll'),
)
def build_reliable_info_packet(sequence_number, ack_number, poll,
port, information):
return ReliableInfoPacket.build(construct.Container(
sequence_number=sequence_number, ack_number=ack_number, poll=poll,
port=port, information=information, length=len(information)+6,
discriminator=None, padding=b''))
def build_reliable_supervisory_packet(
kind, ack_number, poll=False, final=False):
return ReliableSupervisoryPacket.build(construct.Container(
kind=kind, ack_number=ack_number, poll=poll or final,
final=None, reserved=None, discriminator=None))
class ReliableTransport(object):
'''The reliable transport protocol, also known as TRAIN.
The protocol is based on LAPB from ITU-T Recommendation X.25.
'''
NCP_PROTOCOL_NUMBER = 0xBA33
COMMAND_PROTOCOL_NUMBER = 0x3A33
RESPONSE_PROTOCOL_NUMBER = 0x3A35
MODULUS = 128
max_retransmits = 10 # N2 system parameter in LAPB
retransmit_timeout = 0.2 # T1 system parameter
def __init__(self, interface, link_mtu):
self.logger = pulse2_logging.TaggedAdapter(
logger, {'tag': type(self).__name__})
self.send_queue = queue.Queue()
self.opened = threading.Event()
self.closed = False
self.last_sent_packet = None
# The sequence number of the next in-sequence I-packet to be Tx'ed
self.send_variable = 0 # V(S) in LAPB
self.retransmit_count = 0
self.waiting_for_ack = False
self.last_ack_number = 0 # N(R) of the most recently received packet
self.transmit_lock = threading.RLock()
self.retransmit_timer = None
# The expected sequence number of the next received I-packet
self.receive_variable = 0 # V(R) in LAPB
self.sockets = {}
self._mtu = link_mtu - 6
self.command_socket = interface.connect(
self.COMMAND_PROTOCOL_NUMBER)
self.response_socket = interface.connect(
self.RESPONSE_PROTOCOL_NUMBER)
self.command_socket.on_packet = self.command_packet_received
self.response_socket.on_packet = self.response_packet_received
self.ncp = TransportControlProtocol(
interface=interface, transport=self,
ncp_protocol=self.NCP_PROTOCOL_NUMBER,
display_name='ReliableControlProtocol')
self.ncp.up()
self.ncp.open()
@property
def mtu(self):
return self._mtu
def reset_stats(self):
self.stats = {
'info_packets_sent': 0,
'info_packets_received': 0,
'retransmits': 0,
'out_of_order_packets': 0,
'round_trip_time': stats.OnlineStatistics(),
}
self.last_packet_sent_time = None
def this_layer_up(self):
self.send_variable = 0
self.receive_variable = 0
self.retransmit_count = 0
self.last_ack_number = 0
self.waiting_for_ack = False
self.reset_stats()
# We can't let PCMP bind itself using the public open_socket
# method as the method will block until self.opened is set, but
# it won't be set until the peer sends us a packet over the
# transport. But we want to bind the port without waiting.
self.pcmp = pcmp.PulseControlMessageProtocol(
self, pcmp.PulseControlMessageProtocol.PORT)
self.sockets[pcmp.PulseControlMessageProtocol.PORT] = self.pcmp
self.pcmp.on_port_closed = self.on_port_closed
# Send an RR command packet to elicit an RR response from the
# remote peer. Receiving a response from the peer confirms that
# the transport is ready to carry traffic, at which point we
# will allow applications to start opening sockets.
self.send_supervisory_command(kind='RR', poll=True)
self.start_retransmit_timer()
def this_layer_down(self):
self.opened.clear()
if self.retransmit_timer:
self.retransmit_timer.cancel()
self.retransmit_timer = None
self.close_all_sockets()
self.logger.info('Info packets sent=%d retransmits=%d',
self.stats['info_packets_sent'],
self.stats['retransmits'])
self.logger.info('Info packets received=%d out-of-order=%d',
self.stats['info_packets_received'],
self.stats['out_of_order_packets'])
self.logger.info('Round-trip %s ms', self.stats['round_trip_time'])
def open_socket(self, port, timeout=30.0, factory=Socket):
if self.closed:
raise ValueError('Cannot open socket on closed transport')
if port in self.sockets and not self.sockets[port].closed:
raise KeyError('Another socket is already opened '
'on port 0x%04x' % port)
if not self.opened.wait(timeout):
return None
socket = factory(self, port)
self.sockets[port] = socket
return socket
def unregister_socket(self, port):
del self.sockets[port]
def down(self):
self.closed = True
self.close_all_sockets()
self.command_socket.close()
self.response_socket.close()
self.ncp.down()
def close_all_sockets(self):
for socket in list(self.sockets.values()):
socket.close()
self.sockets = {}
def on_port_closed(self, closed_port):
self.logger.info('Remote peer says port 0x%04X is closed; '
'closing socket', closed_port)
try:
self.sockets[closed_port].close()
except KeyError:
self.logger.exception('No socket is open on port 0x%04X!',
closed_port)
def _send_info_packet(self, port, information):
packet = build_reliable_info_packet(
sequence_number=self.send_variable,
ack_number=self.receive_variable,
poll=True, port=port, information=information)
self.command_socket.send(packet)
self.stats['info_packets_sent'] += 1
self.last_packet_sent_time = time.time()
def send(self, port, information):
if self.closed:
raise exceptions.TransportNotReady(
'I/O operation on closed transport')
if not self.opened.is_set():
raise exceptions.TransportNotReady(
'Attempted to send a packet while the reliable transport '
'is not open')
if len(information) > self.mtu:
raise ValueError('Packet length (%d) exceeds transport MTU (%d)' % (
len(information), self.mtu))
self.send_queue.put((port, information))
self.pump_send_queue()
def process_ack(self, ack_number):
with self.transmit_lock:
if not self.waiting_for_ack:
# Could be in the timer recovery condition (waiting for
# a response to an RR Poll command). This is a bit
# hacky and should probably be changed to use an
# explicit state machine when this transport is
# extended to support Go-Back-N ARQ.
if self.retransmit_timer:
self.retransmit_timer.cancel()
self.retransmit_timer = None
self.retransmit_count = 0
if (ack_number - 1) % self.MODULUS == self.send_variable:
if self.retransmit_timer:
self.retransmit_timer.cancel()
self.retransmit_timer = None
self.retransmit_count = 0
self.waiting_for_ack = False
self.send_variable = (self.send_variable + 1) % self.MODULUS
if self.last_packet_sent_time:
self.stats['round_trip_time'].update(
(time.time() - self.last_packet_sent_time) * 1000)
def pump_send_queue(self):
with self.transmit_lock:
if not self.waiting_for_ack:
try:
port, information = self.send_queue.get_nowait()
self.last_sent_packet = (port, information)
self.waiting_for_ack = True
self._send_info_packet(port, information)
self.start_retransmit_timer()
except queue.Empty:
pass
def start_retransmit_timer(self):
if self.retransmit_timer:
self.retransmit_timer.cancel()
self.retransmit_timer = threading.Timer(
self.retransmit_timeout,
self.retransmit_timeout_expired)
self.retransmit_timer.daemon = True
self.retransmit_timer.start()
def retransmit_timeout_expired(self):
with self.transmit_lock:
self.retransmit_count += 1
if self.retransmit_count < self.max_retransmits:
self.stats['retransmits'] += 1
if self.last_sent_packet:
self._send_info_packet(*self.last_sent_packet)
else:
# No info packet to retransmit; must be an RR command
# that needs to be retransmitted.
self.send_supervisory_command(kind='RR', poll=True)
self.start_retransmit_timer()
else:
self.logger.warning('Reached maximum number of retransmit '
'attempts')
self.ncp.restart()
def send_supervisory_command(self, kind, poll=False):
with self.transmit_lock:
command = build_reliable_supervisory_packet(
kind=kind, poll=poll, ack_number=self.receive_variable)
self.command_socket.send(command)
def send_supervisory_response(self, kind, final=False):
with self.transmit_lock:
response = build_reliable_supervisory_packet(
kind=kind, final=final, ack_number=self.receive_variable)
self.response_socket.send(response)
def command_packet_received(self, packet):
if not self.ncp.is_Opened():
self.logger.warning('Received command packet before transport '
'is open. Discarding.')
return
# Information packets have the LSBit of the first byte cleared.
is_info = (bytearray(packet[0])[0] & 0b1) == 0
try:
if is_info:
fields = ReliableInfoPacket.parse(packet)
else:
fields = ReliableSupervisoryPacket.parse(packet)
except (construct.ConstructError, ValueError):
self.logger.exception('Received malformed command packet')
self.ncp.restart()
return
self.opened.set()
if is_info:
if fields.sequence_number == self.receive_variable:
self.receive_variable = (
self.receive_variable + 1) % self.MODULUS
self.stats['info_packets_received'] += 1
if len(fields.information) + 6 == fields.length:
if fields.port in self.sockets:
self.sockets[fields.port].on_receive(
fields.information)
else:
self.logger.warning(
'Received packet on closed port %04X',
fields.port)
else:
self.logger.error(
'Received truncated or corrupt info packet '
'(expected %d data bytes, got %d)',
fields.length-6, len(fields.information))
else:
self.stats['out_of_order_packets'] += 1
self.send_supervisory_response(kind='RR', final=fields.poll)
else:
if fields.kind not in ('RR', 'REJ'):
self.logger.error('Received a %s command packet, which is not '
'yet supported by this implementation',
fields.kind)
# Pretend it is an RR packet
self.process_ack(fields.ack_number)
if fields.poll:
self.send_supervisory_response(kind='RR', final=True)
self.pump_send_queue()
def response_packet_received(self, packet):
if not self.ncp.is_Opened():
self.logger.error(
'Received response packet before transport is open. '
'Discarding.')
return
# Information packets cannot be responses; we only need to
# handle receiving Supervisory packets.
try:
fields = ReliableSupervisoryPacket.parse(packet)
except (construct.ConstructError, ValueError):
self.logger.exception('Received malformed response packet')
self.ncp.restart()
return
self.opened.set()
self.process_ack(fields.ack_number)
self.pump_send_queue()
if fields.kind not in ('RR', 'REJ'):
self.logger.error('Received a %s response packet, which is not '
'yet supported by this implementation.',
fields.kind)