split off structutil, added README, various small fixes

6 files changed, 260 insertions, 197 deletions

diff --git a/README b/README
new file mode 100644
index 0000000..1dc7c44
--- /dev/null
+++ b/README
@@ -0,0 +1,14 @@
+SPICE protocol dumping using demarshaller in spice.
+
+expects to be side by side with spice directory:
+../pyclient/spicedump.py
+../spice/python/python_modules/spice_parser.py
+
+You can try out spicedump like this:
+
+run qemu as usual with spice listening on port 5927 for instance.
+run spicedump as proxy:
+ ./spicedump.py -p -r localhost:5927 -l 8883
+then run spicec using the proxied port:
+ spicec -h localhost -p 8883
+
diff --git a/client.py b/client.py
index dc9d796..aa164cb 100644
--- a/client.py
+++ b/client.py
@@ -1,8 +1,9 @@
-import struct
 import socket
-
 import logging
+import struct
 
+from structutil import (Struct, StructMeta, uint16, uint8, uint64,
+    uint32, uint32_arr, list_to_str)
 import client_proto
 
 logger = logging.getLogger('client')
@@ -31,112 +32,6 @@ SPICE_MAX_PASSWORD_LENGTH=60
 SPICE_TICKET_KEY_PAIR_LENGTH=1024
 SPICE_TICKET_PUBKEY_BYTES=(SPICE_TICKET_KEY_PAIR_LENGTH / 8 + 34)
 
-def unpack_list(structs, s):
-    """ struct has a bug - 
-    sizeof('IBIII') == 20
-    sizeof('IIIIB') == 16
-    """
-    ret = []
-    start = 0
-    for the_struct in structs:
-        ret.extend(list(the_struct.unpack(s[start:start + s.size])))
-        t += s.size
-    return ret
-
-def group(format):
-    return reduce(lambda cs, c: cs[:-1]+[cs[-1]+c] if len(cs) > 0 and c == cs[-1][-1] else cs+[c], format, [])
-
-class Elements(object):
-    pass
-
-ENDIANESS = '<' # small endian
-
-class StructList(object):
-
-    def __init__(self, formats):
-        self._s = map(struct.Struct, (ENDIANESS+f for f in formats))
-        self.size = sum([s.size for s in self._s])
-
-    def pack(self, *args):
-        i_s, i_e = 0, 0
-        r = []
-        for s in self._s:
-            i_e += len(s.format) - (1 if s.format[0] in '<>' else 0)
-            r.append(s.pack(*args[i_s:i_e]))
-            i_s = i_e
-        return ''.join(r)
-
-    def unpack(self, st):
-        r = []
-        i_s, i_e = 0, 0
-        for s in self._s:
-            i_e += s.size
-            r.append(list(s.unpack(st[i_s:i_e])))
-            i_s = i_e
-        return sum(r, [])
-
-class StructMeta(type):
-    def __new__(meta, classname, bases, classDict):
-        fields = classDict['fields']
-        is_complex = classDict['_is_complex'] = callable(fields[-1][0])
-        if is_complex:
-            classDict['complex_field'] = complex_field = fields[-1]
-            fields = fields[:-1]
-        assert(not any(map(callable, fields)))
-        classDict['_s'] = StructList(group(''.join(t for t,n in fields)))
-        classDict['_names'] = [n for t,n in fields]
-        classDict['size'] = classDict['_s'].size
-        classDict['field_elements'] = [len(t) for t, n in fields]
-        return type.__new__(meta, classname, bases, classDict)
-
-def indice_pairs(sizes):
-    s = 0
-    for size in sizes:
-        yield s, s+size
-        s += size
-
-def cut(elements, sizes):
-    for s, e in indice_pairs(sizes):
-        if e - s == 1:
-            yield elements[s]
-        else:
-            yield elements[s:e]
-
-class Struct(object):
-    @classmethod
-    def parse(cls, s):
-        base = list(cut(cls._s.unpack(s[:cls._s.size]), cls.field_elements))
-        if cls._is_complex:
-            import pdb; pdb.set_trace()
-            return base
-        return base
-
-    @classmethod
-    def make(cls, **kw):
-        args = []
-        args = [kw[n] for n in cls._names]
-        assert(len(args) == len(cls._names) == len(kw))
-        return cls._s.pack(*args)
-
-    def __init__(self, *args, **kw):
-        self.e = Elements()
-        if (len(kw) == 0 and len(args) == 1) or (len(kw) == 1 and kw.has_key('s')):
-            s = args[0] if len(args) == 1 else kw['s']
-            self.elements = elements = self.parse(s)
-        else:
-            self.elements = elements = [kw[n] for n in self._names]
-        self.e.__dict__.update(dict(zip(self._names, elements)))
-
-    def tostr(self):
-        return self.make(**self.e.__dict__)
-
-uint16 = 'H'
-uint32 = 'I'
-uint64 = 'Q'
-uint8 = 'B'
-
-uint32_arr = lambda s: ENDIANESS + uint32*s.e.size
-
 class SpiceDataHeader(Struct):
     __metaclass__ = StructMeta
     fields = [(uint64, 'serial'), (uint16, 'type'), (uint32, 'size'),
@@ -240,11 +135,6 @@ def connect(host, port):
     s.connect((host, port))
     return s
 
-def list_to_str(l, type=uint32):
-    if len(l) == 0:
-        return ''
-    return struct.pack(ENDIANESS+len(l)*type, l)
-
 class Channel(object):
     def __init__(self, s, connection_id, channel_type, channel_id,
             common_caps, channel_caps):
diff --git a/dumpspice.py b/dumpspice.py
deleted file mode 100755
index 9c643c9..0000000
--- a/dumpspice.py
+++ /dev/null
@@ -1,83 +0,0 @@
-#!/usr/bin/env python
-import sys
-import pcaputil
-from proxy import proxy, closeallsockets
-from collections import defaultdict
-from time import time
-from select import select
-from optparse import OptionParser
-import logging
-
-dt = 1.0
-
-class Histogram(defaultdict):
-    def __init__(self):
-        super(Histogram, self).__init__(lambda: (0,0))
-        self.last = defaultdict(lambda: (0,0))
-    def show(self):
-        print "----------------------------"
-        print '\n'.join(['%20s: %6d %4d' % (k, self[k][1], self[k][1] - self.last[k][1]) for t,k in sorted((t,k) for k,(t,c) in self.items())])
-        self.last.update(self)
-
-verbose = 0
-
-def dumpspice(p, stdscr=None):
-    import pcapspice
-    spice = pcapspice.spice_iter(p)
-    hist = Histogram()
-    last_print = start_time = time()
-    if stdscr:
-        stdscr.erase()
-    # replace the "for d in spice:" loop with a select
-    while True:
-        #rds, _ws, _xs = select([p.fileno()],[],[],dt)
-        cur_time = time()
-        do_read = True # = len(rds) > 0:
-        if do_read:
-            d = spice.next()
-            if verbose:
-                print d
-            old_time, old_count = hist[d.msg.data.result_name]
-            hist[d.msg.data.result_name] = (cur_time, old_count + 1)
-        if cur_time - last_print > dt:
-            hist.show()
-            last_print = cur_time
-
-def frompcap(stdscr=None):
-    p = pcaputil.packet_iter('lo')
-    return dumpspice(p, stdscr)
-
-def fromproxy(stdscr, local_port, remote_addr):
-    p = proxy(local_port=local_port, remote_addr=remote_addr)
-    return dumpspice(p, stdscr=stdscr)
-
-if __name__ == '__main__':
-    parser = OptionParser()
-    parser.add_option('-p', '--proxy', dest='proxy', help='use proxy',
-        action='store_true')
-    parser.add_option('-l', '--localport', dest='local_port', help='set proxy local port')
-    parser.add_option('-r', '--remoteaddr', dest='remote_addr', help='set proxy remote address')
-    parser.add_option('-v', '--verbose', dest='verbose', action='count', help='verbosity', default=0)
-    parser.add_option('-c', '--curses', dest='curses', action='store_true', help='use curses')
-    opts, rest = parser.parse_args(sys.argv[1:])
-    if opts.verbose >= 2 in sys.argv:
-        logging.basicConfig(filename='dumpspice.log', level=logging.DEBUG)
-        print "saving debug log to dumpspice.log"
-    if opts.proxy:
-        local_port = int(opts.local_port)
-        remote_addr = opts.remote_addr.split(':')
-        remote_addr = (remote_addr[0], int(remote_addr[1]))
-        main = (lambda stdscr, local_port=local_port, remote_addr=remote_addr:
-                    fromproxy(stdscr, local_port, remote_addr))
-    else:
-        main = frompcap
-    verbose = opts.verbose
-    try:
-        if opts.curses in sys.argv:
-            import curses
-            curses.wrapper(main)
-        else:
-            main(None)
-    except KeyboardInterrupt, e:
-         closeallsockets()
-
diff --git a/pcapspice.py b/pcapspice.py
index 05a5bef..29c7027 100644
--- a/pcapspice.py
+++ b/pcapspice.py
@@ -3,7 +3,8 @@ import logging
 
 from pcaputil import header_conversation_iter
 import client_proto
-from client import SpiceLinkHeader, SpiceLinkMess, SpiceLinkReply, SpiceDataHeader
+from client import (SpiceLinkHeader, SpiceLinkMess,
+                    SpiceLinkReply, SpiceDataHeader)
 
 logger = logging.getLogger('pcapspice')
 
diff --git a/spicedump.py b/spicedump.py
new file mode 100755
index 0000000..a37b2f6
--- /dev/null
+++ b/spicedump.py
@@ -0,0 +1,127 @@
+#!/usr/bin/env python
+import sys
+import pcaputil
+from proxy import proxy, closeallsockets
+from collections import defaultdict
+from itertools import izip_longest
+from time import time
+from select import select
+from optparse import OptionParser
+import logging
+
+dt = 1.0
+verbose = 0
+
+class Histogram(defaultdict):
+    def __init__(self):
+        super(Histogram, self).__init__(lambda: (0,0))
+        self.last = defaultdict(lambda: (0,0))
+    def show(self):
+        for t,k in sorted((t,k) for k,(t,c) in self.items()):
+            diff = self[k][1] - self.last[k][1]
+            yield '%20s: %6d %4d' % (k, self[k][1], diff)
+        self.last.update(self)
+
+def linesmerge(*line_sources):
+    for same_height_lines in izip_longest(*line_sources):
+        yield ''.join([x for x in same_height_lines if x is not None])
+
+def show(*line_sources):
+    print "------------------------------------------------"
+    for line in linesmerge(*line_sources):
+        print line
+
+class SurfaceStatistics(object):
+    def __init__(self):
+        self.s = defaultdict(lambda: defaultdict(lambda: 0))
+    def add(self, sid, op):
+        self.s[sid][op] += 1
+    def show(self):
+        all_surfaces = self.s.keys()
+        ds = self.s.values()
+        ops = set(sum([d.keys() for d in ds], []))
+        per_surface = dict([(o,
+            sorted(set([
+                sid for sid, d in self.s.items() if d[o] > 0
+                       ]))) for o in ops])
+        for k, v in per_surface.items():
+            if k == 'stream_create' and len(v) > 1:
+                import pdb; pdb.set_trace()
+            yield '%20s: %s' % (k, ','.join(map(str, v)))
+
+def spicedump(p, stdscr=None):
+    import pcapspice
+    spice = pcapspice.spice_iter(p)
+    hist = Histogram()
+    surface_stat = SurfaceStatistics()
+    last_print = start_time = time()
+    messages = ['welcome to spicedump (spice quest?)']
+    if stdscr:
+        stdscr.erase()
+    # replace the "for d in spice:" loop with a select
+    while True:
+        #rds, _ws, _xs = select([p.fileno()],[],[],dt)
+        cur_time = time()
+        do_read = True # = len(rds) > 0:
+        if do_read:
+            d = spice.next()
+            result_name = d.msg.data.result_name
+            result_value = d.msg.data.result_value
+            if (any(x in result_name for x in ['surface', 'stream'])
+                and not result_name in
+                    ['stream_data', 'stream_clip', 'stream_destroy']):
+                msg_d = dict(result_value)
+                if 'surface_id' in msg_d:
+                    surface_stat.add(msg_d['surface_id'], result_name)
+                    if msg_d['surface_id'] != 0 and 'stream' in result_name:
+                        messages.append("non zero surface id in stream: %s" % result_value)
+                else:
+                    import pdb; pdb.set_trace()
+            if verbose:
+                messages.extend(str(d).split('\n'))
+            old_time, old_count = hist[result_name]
+            hist[result_name] = (cur_time, old_count + 1)
+        if cur_time - last_print > dt:
+            show(hist.show(), surface_stat.show())
+            print '\n'.join(messages[-20:])
+            last_print = cur_time
+
+def frompcap(stdscr=None):
+    p = pcaputil.packet_iter('lo')
+    return spicedump(p, stdscr)
+
+def fromproxy(stdscr, local_port, remote_addr):
+    p = proxy(local_port=local_port, remote_addr=remote_addr)
+    return spicedump(p, stdscr=stdscr)
+
+if __name__ == '__main__':
+    parser = OptionParser()
+    parser.add_option('-p', '--proxy', dest='proxy', help='use proxy',
+        action='store_true')
+    parser.add_option('-l', '--localport', dest='local_port', help='set proxy local port')
+    parser.add_option('-r', '--remoteaddr', dest='remote_addr', help='set proxy remote address')
+    parser.add_option('-v', '--verbose', dest='verbose', action='count', help='verbosity', default=0)
+    parser.add_option('-c', '--curses', dest='curses', action='store_true', help='use curses')
+    opts, rest = parser.parse_args(sys.argv[1:])
+    if opts.verbose >= 2 in sys.argv:
+        logging.basicConfig(filename='spicedump.log', level=logging.DEBUG)
+        print "saving debug log to spicedump.log"
+    if opts.proxy:
+        local_port = int(opts.local_port)
+        remote_addr = opts.remote_addr.split(':')
+        remote_addr = (remote_addr[0], int(remote_addr[1]))
+        main = (lambda stdscr, local_port=local_port, remote_addr=remote_addr:
+                    fromproxy(stdscr, local_port, remote_addr))
+    else:
+        main = frompcap
+    verbose = opts.verbose
+    try:
+        if opts.curses in sys.argv:
+            import curses
+            curses.wrapper(main)
+        else:
+            main(None)
+    except KeyboardInterrupt, e:
+        # XXX - ctrl-c doesn't reach here :(
+        closeallsockets()
+
diff --git a/structutil.py b/structutil.py
new file mode 100644
index 0000000..7a26e86
--- /dev/null
+++ b/structutil.py
@@ -0,0 +1,114 @@
+import struct
+
+ENDIANESS = '<' # small endian
+
+uint16 = 'H'
+uint32 = 'I'
+uint64 = 'Q'
+uint8 = 'B'
+
+uint32_arr = lambda s: ENDIANESS + uint32*s.e.size
+
+def unpack_list(structs, s):
+    """ struct has a bug - 
+    sizeof('IBIII') == 20
+    sizeof('IIIIB') == 16
+    """
+    ret = []
+    start = 0
+    for the_struct in structs:
+        ret.extend(list(the_struct.unpack(s[start:start + s.size])))
+        t += s.size
+    return ret
+
+def group(format):
+    return reduce(lambda cs, c: cs[:-1]+[cs[-1]+c] if len(cs) > 0 and c == cs[-1][-1] else cs+[c], format, [])
+
+class Elements(object):
+    pass
+
+class StructList(object):
+
+    def __init__(self, formats):
+        self._s = map(struct.Struct, (ENDIANESS+f for f in formats))
+        self.size = sum([s.size for s in self._s])
+
+    def pack(self, *args):
+        i_s, i_e = 0, 0
+        r = []
+        for s in self._s:
+            i_e += len(s.format) - (1 if s.format[0] in '<>' else 0)
+            r.append(s.pack(*args[i_s:i_e]))
+            i_s = i_e
+        return ''.join(r)
+
+    def unpack(self, st):
+        r = []
+        i_s, i_e = 0, 0
+        for s in self._s:
+            i_e += s.size
+            r.append(list(s.unpack(st[i_s:i_e])))
+            i_s = i_e
+        return sum(r, [])
+
+class StructMeta(type):
+    def __new__(meta, classname, bases, classDict):
+        fields = classDict['fields']
+        is_complex = classDict['_is_complex'] = callable(fields[-1][0])
+        if is_complex:
+            classDict['complex_field'] = complex_field = fields[-1]
+            fields = fields[:-1]
+        assert(not any(map(callable, fields)))
+        classDict['_s'] = StructList(group(''.join(t for t,n in fields)))
+        classDict['_names'] = [n for t,n in fields]
+        classDict['size'] = classDict['_s'].size
+        classDict['field_elements'] = [len(t) for t, n in fields]
+        return type.__new__(meta, classname, bases, classDict)
+
+def slice_pairs_iter(sizes):
+    s = 0
+    for size in sizes:
+        yield s, s+size
+        s += size
+
+def cut(elements, sizes):
+    for s, e in slice_pairs_iter(sizes):
+        if e - s == 1:
+            yield elements[s]
+        else:
+            yield elements[s:e]
+
+class Struct(object):
+    @classmethod
+    def parse(cls, s):
+        base = list(cut(cls._s.unpack(s[:cls._s.size]), cls.field_elements))
+        if cls._is_complex:
+            import pdb; pdb.set_trace()
+            return base
+        return base
+
+    @classmethod
+    def make(cls, **kw):
+        args = []
+        args = [kw[n] for n in cls._names]
+        assert(len(args) == len(cls._names) == len(kw))
+        return cls._s.pack(*args)
+
+    def __init__(self, *args, **kw):
+        self.e = Elements()
+        if (len(kw) == 0 and len(args) == 1) or (len(kw) == 1 and kw.has_key('s')):
+            s = args[0] if len(args) == 1 else kw['s']
+            self.elements = elements = self.parse(s)
+        else:
+            self.elements = elements = [kw[n] for n in self._names]
+        self.e.__dict__.update(dict(zip(self._names, elements)))
+
+    def tostr(self):
+        return self.make(**self.e.__dict__)
+
+def list_to_str(l, type=uint32):
+    if len(l) == 0:
+        return ''
+    return struct.pack(ENDIANESS+len(l)*type, l)
+
+