Rev 4768: A bit broken, but getting there. in http://bazaar.launchpad.net/~jameinel/bzr/2.1-static-tuple-chk-map
John Arbash Meinel
john at arbash-meinel.com
Tue Oct 20 23:13:37 BST 2009
At http://bazaar.launchpad.net/~jameinel/bzr/2.1-static-tuple-chk-map
------------------------------------------------------------
revno: 4768
revision-id: john at arbash-meinel.com-20091020221323-vvukgazqxkicb70n
parent: john at arbash-meinel.com-20091020194646-wnqpd15qs19y28z7
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: 2.1-static-tuple-chk-map
timestamp: Tue 2009-10-20 17:13:23 -0500
message:
A bit broken, but getting there.
Start being much stricter about requiring StaticTuples everywhere.
I may go back and loosen this restriction, but getting the code base
StaticTuple pure is probably a good idea. The main reason to be 'looser'
is so that things don't fail 'in the wild' just because someone
calls an api with a tuple rather than a StaticTuple.
However, I'd like the internals to be 'pure' if possible.
We'll see.
-------------- next part --------------
=== modified file 'bzrlib/_chk_map_py.py'
--- a/bzrlib/_chk_map_py.py 2009-10-08 16:01:53 +0000
+++ b/bzrlib/_chk_map_py.py 2009-10-20 22:13:23 +0000
@@ -19,6 +19,8 @@
import zlib
import struct
+from bzrlib.static_tuple import StaticTuple
+
_LeafNode = None
_InternalNode = None
_unknown = None
@@ -93,7 +95,7 @@
value_lines = lines[pos:pos+num_value_lines]
pos += num_value_lines
value = '\n'.join(value_lines)
- items[tuple(elements[:-1])] = value
+ items[StaticTuple(*elements[:-1])] = value
if len(items) != length:
raise AssertionError("item count (%d) mismatch for key %s,"
" bytes %r" % (length, key, bytes))
@@ -141,7 +143,7 @@
for line in lines[5:]:
line = common_prefix + line
prefix, flat_key = line.rsplit('\x00', 1)
- items[prefix] = (flat_key,)
+ items[prefix] = StaticTuple(flat_key,)
if len(items) == 0:
raise AssertionError("We didn't find any item for %s" % key)
result._items = items
@@ -155,6 +157,3 @@
result._node_width = len(prefix)
result._search_prefix = common_prefix
return result
-
-
-_key_type = tuple
=== modified file 'bzrlib/_chk_map_pyx.pyx'
--- a/bzrlib/_chk_map_pyx.pyx 2009-10-08 16:01:53 +0000
+++ b/bzrlib/_chk_map_pyx.pyx 2009-10-20 22:13:23 +0000
@@ -103,8 +103,8 @@
cdef char *c_out
# cdef PyObject *bit
- if not PyTuple_CheckExact(key) and not StaticTuple_CheckExact(key):
- raise TypeError('key %r is not a tuple' % (key,))
+ if not StaticTuple_CheckExact(key):
+ raise TypeError('key %r is not a StaticTuple' % (key,))
num_bits = len(key)
# 4 bytes per crc32, and another 1 byte between bits
num_out_bytes = (9 * num_bits) - 1
@@ -143,8 +143,8 @@
cdef char *c_out
# cdef PyObject *bit
- if not PyTuple_CheckExact(key) and not StaticTuple_CheckExact(key):
- raise TypeError('key %r is not a tuple' % (key,))
+ if not StaticTuple_CheckExact(key):
+ raise TypeError('key %r is not a StaticTuple' % (key,))
num_bits = len(key)
# 4 bytes per crc32, and another 1 byte between bits
num_out_bytes = (5 * num_bits) - 1
@@ -282,6 +282,8 @@
cur = next_line + 1
entry_bits = StaticTuple_New(width)
for i from 0 <= i < num_prefix_bits:
+ # TODO: Use PyList_GetItem, or turn prefix_bits into a
+ # tuple/StaticTuple
entry = prefix_bits[i]
# SET_ITEM 'steals' a reference
Py_INCREF(entry)
@@ -359,6 +361,8 @@
_unknown = chk_map._unknown
result = _InternalNode(search_key_func=search_key_func)
+ if not StaticTuple_CheckExact(key):
+ raise TypeError('key %r is not a StaticTuple' % (key,))
if not PyString_CheckExact(bytes):
raise TypeError('bytes must be a plain string not %s' % (type(bytes),))
@@ -415,6 +419,3 @@
result._node_width = len(item_prefix)
result._search_prefix = PyString_FromStringAndSize(prefix, prefix_length)
return result
-
-
-_key_type = StaticTuple
=== modified file 'bzrlib/_static_tuple_c.c'
--- a/bzrlib/_static_tuple_c.c 2009-10-17 00:34:28 +0000
+++ b/bzrlib/_static_tuple_c.c 2009-10-20 22:13:23 +0000
@@ -250,8 +250,9 @@
obj = PyTuple_GET_ITEM(args, i);
if (!PyString_CheckExact(obj)) {
if (!StaticTuple_CheckExact(obj)) {
- PyErr_SetString(PyExc_TypeError, "StaticTuple.__init__(...)"
- " requires that all items are strings or StaticTuple.");
+ PyErr_Format(PyExc_TypeError, "StaticTuple.__init__(...)"
+ " requires that all items are strings or StaticTuple"
+ " not %s", Py_TYPE(obj)->tp_name);
type->tp_dealloc((PyObject *)self);
return NULL;
}
=== modified file 'bzrlib/chk_map.py'
--- a/bzrlib/chk_map.py 2009-10-08 16:01:53 +0000
+++ b/bzrlib/chk_map.py 2009-10-20 22:13:23 +0000
@@ -52,6 +52,7 @@
registry,
trace,
)
+from bzrlib.static_tuple import StaticTuple
# approx 4MB
# If each line is 50 bytes, and you have 255 internal pages, with 255-way fan
@@ -100,6 +101,7 @@
if root_key is None:
self._root_node = LeafNode(search_key_func=search_key_func)
else:
+ _check_key(root_key)
self._root_node = self._node_key(root_key)
def apply_delta(self, delta):
@@ -133,7 +135,7 @@
def _ensure_root(self):
"""Ensure that the root node is an object not a key."""
- if type(self._root_node) in (_key_type, tuple):
+ if type(self._root_node) is StaticTuple:
# Demand-load the root
self._root_node = self._get_node(self._root_node)
@@ -147,7 +149,7 @@
:param node: A tuple key or node object.
:return: A node object.
"""
- if type(node) in (tuple, _key_type):
+ if type(node) is StaticTuple:
bytes = self._read_bytes(node)
return _deserialise(bytes, node,
search_key_func=self._search_key_func)
@@ -218,6 +220,7 @@
root_key = klass._create_directly(store, initial_value,
maximum_size=maximum_size, key_width=key_width,
search_key_func=search_key_func)
+ assert type(root_key) is StaticTuple
return root_key
@classmethod
@@ -256,7 +259,11 @@
for split, subnode in node_details:
node.add_node(split, subnode)
keys = list(node.serialise(store))
- return keys[-1]
+ root_node = keys[-1]
+ assert (type(root_node) is StaticTuple
+ and len(root_node) == 1 and
+ type(root_node[0]) is str)
+ return root_node
def iter_changes(self, basis):
"""Iterate over the changes between basis and self.
@@ -486,7 +493,8 @@
def key(self):
"""Return the key for this map."""
- if type(self._root_node) in (tuple, _key_type):
+ if type(self._root_node) is StaticTuple:
+ _check_key(self._root_node)
return self._root_node
else:
return self._root_node._key
@@ -516,8 +524,12 @@
def _node_key(self, node):
"""Get the key for a node whether it's a tuple or node."""
- if type(node) in (tuple, _key_type):
+ if type(node) is StaticTuple:
+ _check_key(node)
return node
+ elif type(node) is tuple:
+ raise TypeError('node %r should be a StaticTuple not tuple'
+ % (node,))
else:
return node._key
@@ -542,7 +554,7 @@
:return: The key of the root node.
"""
- if type(self._root_node) in (tuple, _key_type):
+ if type(self._root_node) is StaticTuple:
# Already saved.
return self._root_node
keys = list(self._root_node.serialise(self._store))
@@ -873,7 +885,7 @@
lines.append(serialized[prefix_len:])
lines.extend(value_lines)
sha1, _, _ = store.add_lines((None,), (), lines)
- self._key = ("sha1:" + sha1,)
+ self._key = StaticTuple("sha1:" + sha1,).intern()
bytes = ''.join(lines)
if len(bytes) != self._current_size():
raise AssertionError('Invalid _current_size')
@@ -994,6 +1006,9 @@
:param key: The key that the serialised node has.
:return: An InternalNode instance.
"""
+ if type(key) is not StaticTuple:
+ import pdb; pdb.set_trace()
+ key = StaticTuple.from_sequence(key).intern()
return _deserialise_internal_node(bytes, key,
search_key_func=search_key_func)
@@ -1024,7 +1039,7 @@
# for whatever we are missing
shortcut = True
for prefix, node in self._items.iteritems():
- if node.__class__ in (tuple, _key_type):
+ if node.__class__ is StaticTuple:
keys[node] = (prefix, None)
else:
yield node, None
@@ -1059,7 +1074,7 @@
# A given key can only match 1 child node, if it isn't
# there, then we can just return nothing
return
- if node.__class__ in (tuple, _key_type):
+ if node.__class__ is StaticTuple:
keys[node] = (search_prefix, [key])
else:
# This is loaded, and the only thing that can match,
@@ -1092,7 +1107,7 @@
# We can ignore this one
continue
node_key_filter = prefix_to_keys[search_prefix]
- if node.__class__ in (tuple, _key_type):
+ if node.__class__ is StaticTuple:
keys[node] = (search_prefix, node_key_filter)
else:
yield node, node_key_filter
@@ -1107,7 +1122,7 @@
if sub_prefix in length_filter:
node_key_filter.extend(prefix_to_keys[sub_prefix])
if node_key_filter: # this key matched something, yield it
- if node.__class__ in (tuple, _key_type):
+ if node.__class__ is StaticTuple:
keys[node] = (prefix, node_key_filter)
else:
yield node, node_key_filter
@@ -1245,7 +1260,7 @@
:return: An iterable of the keys inserted by this operation.
"""
for node in self._items.itervalues():
- if type(node) in (tuple, _key_type):
+ if type(node) is StaticTuple:
# Never deserialised.
continue
if node._key is not None:
@@ -1262,7 +1277,7 @@
lines.append('%s\n' % (self._search_prefix,))
prefix_len = len(self._search_prefix)
for prefix, node in sorted(self._items.items()):
- if type(node) in (tuple, _key_type):
+ if type(node) is StaticTuple:
key = node[0]
else:
key = node._key[0]
@@ -1272,7 +1287,7 @@
% (serialised, self._search_prefix))
lines.append(serialised[prefix_len:])
sha1, _, _ = store.add_lines((None,), (), lines)
- self._key = ("sha1:" + sha1,)
+ self._key = StaticTuple("sha1:" + sha1,).intern()
_page_cache.add(self._key, ''.join(lines))
yield self._key
@@ -1307,7 +1322,7 @@
raise AssertionError("unserialised nodes have no refs.")
refs = []
for value in self._items.itervalues():
- if type(value) in (tuple, _key_type):
+ if type(value) is StaticTuple:
refs.append(value)
else:
refs.append(value.key())
@@ -1639,7 +1654,6 @@
_search_key_255,
_deserialise_leaf_node,
_deserialise_internal_node,
- _key_type,
)
except ImportError, e:
osutils.failed_to_load_extension(e)
@@ -1648,7 +1662,19 @@
_search_key_255,
_deserialise_leaf_node,
_deserialise_internal_node,
- _key_type,
)
search_key_registry.register('hash-16-way', _search_key_16)
search_key_registry.register('hash-255-way', _search_key_255)
+
+def _check_key(key):
+ if type(key) is not StaticTuple:
+ raise TypeError('key %r is not StaticTuple but %s' % (key, type(key)))
+ if len(key) != 1:
+ raise ValueError('key %r should have length 1, not %d' % (key, len(key),))
+ if type(key[0]) is not str:
+ raise TypeError('key %r should hold a str, not %r'
+ % (key, type(key[0])))
+ if not key[0].startswith('sha1:'):
+ raise ValueError('key %r should point to a sha1:' % (key,))
+
+
=== modified file 'bzrlib/inventory.py'
--- a/bzrlib/inventory.py 2009-10-14 13:47:28 +0000
+++ b/bzrlib/inventory.py 2009-10-20 22:13:23 +0000
@@ -51,6 +51,7 @@
)
from bzrlib.symbol_versioning import deprecated_in, deprecated_method
from bzrlib.trace import mutter
+from bzrlib.static_tuple import StaticTuple
class InventoryEntry(object):
@@ -1810,21 +1811,24 @@
pass
deletes.add(file_id)
else:
- new_key = (file_id,)
+ new_key = StaticTuple(file_id,)
new_value = result._entry_to_bytes(entry)
# Update caches. It's worth doing this whether
# we're propagating the old caches or not.
result._path_to_fileid_cache[new_path] = file_id
- parents.add((split(new_path)[0], entry.parent_id))
+ if entry.parent_id is not None:
+ parents.add(StaticTuple(split(new_path)[0].encode('utf8'),
+ entry.parent_id))
if old_path is None:
old_key = None
else:
- old_key = (file_id,)
+ old_key = StaticTuple(file_id,)
if self.id2path(file_id) != old_path:
raise errors.InconsistentDelta(old_path, file_id,
"Entry was at wrong other path %r." %
self.id2path(file_id))
altered.add(file_id)
+ # TODO: use a StaticTuple here, though a value may be None
id_to_entry_delta.append((old_key, new_key, new_value))
if result.parent_id_basename_to_file_id is not None:
# parent_id, basename changes
@@ -1923,7 +1927,13 @@
search_key_name = info.get('search_key_name', 'plain')
parent_id_basename_to_file_id = info.get(
'parent_id_basename_to_file_id', None)
+ if not parent_id_basename_to_file_id.startswith('sha1:'):
+ raise ValueError('parent_id_basename_to_file_id should be a sha1'
+ ' key not %r' % (parent_id_basename_to_file_id,))
id_to_entry = info['id_to_entry']
+ if not id_to_entry.startswith('sha1:'):
+ raise ValueError('id_to_entry should be a sha1'
+ ' key not %r' % (id_to_entry,))
result = CHKInventory(search_key_name)
result.revision_id = revision_id
@@ -1932,12 +1942,13 @@
result._search_key_name)
if parent_id_basename_to_file_id is not None:
result.parent_id_basename_to_file_id = chk_map.CHKMap(
- chk_store, (parent_id_basename_to_file_id,),
+ chk_store, StaticTuple(parent_id_basename_to_file_id,),
search_key_func=search_key_func)
else:
result.parent_id_basename_to_file_id = None
- result.id_to_entry = chk_map.CHKMap(chk_store, (id_to_entry,),
+ result.id_to_entry = chk_map.CHKMap(chk_store,
+ StaticTuple(id_to_entry,),
search_key_func=search_key_func)
if (result.revision_id,) != expected_revision_id:
raise ValueError("Mismatched revision id and expected: %r, %r" %
@@ -1965,7 +1976,8 @@
id_to_entry_dict = {}
parent_id_basename_dict = {}
for path, entry in inventory.iter_entries():
- id_to_entry_dict[(entry.file_id,)] = entry_to_bytes(entry)
+ key = StaticTuple(entry.file_id,).intern()
+ id_to_entry_dict[key] = entry_to_bytes(entry)
p_id_key = parent_id_basename_key(entry)
parent_id_basename_dict[p_id_key] = entry.file_id
@@ -1994,7 +2006,7 @@
parent_id = entry.parent_id
else:
parent_id = ''
- return parent_id, entry.name.encode('utf8')
+ return StaticTuple(parent_id, entry.name.encode('utf8')).intern()
def __getitem__(self, file_id):
"""map a single file_id -> InventoryEntry."""
@@ -2215,9 +2227,9 @@
lines.append('search_key_name: %s\n' % (self._search_key_name,))
lines.append("root_id: %s\n" % self.root_id)
lines.append('parent_id_basename_to_file_id: %s\n' %
- self.parent_id_basename_to_file_id.key())
+ (self.parent_id_basename_to_file_id.key()[0],))
lines.append("revision_id: %s\n" % self.revision_id)
- lines.append("id_to_entry: %s\n" % self.id_to_entry.key())
+ lines.append("id_to_entry: %s\n" % (self.id_to_entry.key()[0],))
else:
lines.append("revision_id: %s\n" % self.revision_id)
lines.append("root_id: %s\n" % self.root_id)
=== modified file 'bzrlib/repofmt/groupcompress_repo.py'
--- a/bzrlib/repofmt/groupcompress_repo.py 2009-10-19 16:21:20 +0000
+++ b/bzrlib/repofmt/groupcompress_repo.py 2009-10-20 22:13:23 +0000
@@ -53,7 +53,9 @@
ResumedPack,
Packer,
)
+from bzrlib.static_tuple import StaticTuple
+_check_key = chk_map._check_key
class GCPack(NewPack):
@@ -814,14 +816,16 @@
' no new_path %r' % (file_id,))
if new_path == '':
new_inv.root_id = file_id
- parent_id_basename_key = ('', '')
+ parent_id_basename_key = StaticTuple('', '')
else:
utf8_entry_name = entry.name.encode('utf-8')
- parent_id_basename_key = (entry.parent_id, utf8_entry_name)
+ parent_id_basename_key = StaticTuple(entry.parent_id,
+ utf8_entry_name)
new_value = entry_to_bytes(entry)
# Populate Caches?
# new_inv._path_to_fileid_cache[new_path] = file_id
- id_to_entry_dict[(file_id,)] = new_value
+ key = StaticTuple(file_id).intern()
+ id_to_entry_dict[key] = new_value
parent_id_basename_dict[parent_id_basename_key] = file_id
new_inv._populate_from_dicts(self.chk_bytes, id_to_entry_dict,
@@ -1170,6 +1174,8 @@
for inv in repo.iter_inventories(inventory_ids, 'unordered'):
root_key = inv.id_to_entry.key()
pid_root_key = inv.parent_id_basename_to_file_id.key()
+ _check_key(root_key)
+ _check_key(pid_root_key)
if inv.revision_id in parent_only_inv_ids:
result.uninteresting_root_keys.add(root_key)
result.uninteresting_pid_root_keys.add(pid_root_key)
=== modified file 'bzrlib/tests/test__chk_map.py'
--- a/bzrlib/tests/test__chk_map.py 2009-04-09 20:23:07 +0000
+++ b/bzrlib/tests/test__chk_map.py 2009-10-20 22:13:23 +0000
@@ -20,6 +20,8 @@
chk_map,
tests,
)
+from bzrlib.static_tuple import StaticTuple
+stuple = StaticTuple
def load_tests(standard_tests, module, loader):
@@ -67,25 +69,25 @@
self.assertEqual(expected, actual, 'actual: %r' % (actual,))
def test_simple_16(self):
- self.assertSearchKey16('8C736521', ('foo',))
- self.assertSearchKey16('8C736521\x008C736521', ('foo', 'foo'))
- self.assertSearchKey16('8C736521\x0076FF8CAA', ('foo', 'bar'))
- self.assertSearchKey16('ED82CD11', ('abcd',))
+ self.assertSearchKey16('8C736521', stuple('foo',))
+ self.assertSearchKey16('8C736521\x008C736521', stuple('foo', 'foo'))
+ self.assertSearchKey16('8C736521\x0076FF8CAA', stuple('foo', 'bar'))
+ self.assertSearchKey16('ED82CD11', stuple('abcd',))
def test_simple_255(self):
- self.assertSearchKey255('\x8cse!', ('foo',))
- self.assertSearchKey255('\x8cse!\x00\x8cse!', ('foo', 'foo'))
- self.assertSearchKey255('\x8cse!\x00v\xff\x8c\xaa', ('foo', 'bar'))
+ self.assertSearchKey255('\x8cse!', stuple('foo',))
+ self.assertSearchKey255('\x8cse!\x00\x8cse!', stuple('foo', 'foo'))
+ self.assertSearchKey255('\x8cse!\x00v\xff\x8c\xaa', stuple('foo', 'bar'))
# The standard mapping for these would include '\n', so it should be
# mapped to '_'
- self.assertSearchKey255('\xfdm\x93_\x00P_\x1bL', ('<', 'V'))
+ self.assertSearchKey255('\xfdm\x93_\x00P_\x1bL', stuple('<', 'V'))
def test_255_does_not_include_newline(self):
# When mapping via _search_key_255, we should never have the '\n'
# character, but all other 255 values should be present
chars_used = set()
for char_in in range(256):
- search_key = self.module._search_key_255((chr(char_in),))
+ search_key = self.module._search_key_255(stuple(chr(char_in),))
chars_used.update(search_key)
all_chars = set([chr(x) for x in range(256)])
unused_chars = all_chars.symmetric_difference(chars_used)
@@ -113,10 +115,11 @@
def test_deserialise_empty(self):
node = self.module._deserialise_leaf_node(
- "chkleaf:\n10\n1\n0\n\n", ("sha1:1234",))
+ "chkleaf:\n10\n1\n0\n\n", stuple("sha1:1234",))
self.assertEqual(0, len(node))
self.assertEqual(10, node.maximum_size)
self.assertEqual(("sha1:1234",), node.key())
+ self.assertIsInstance(node.key(), StaticTuple)
self.assertIs(None, node._search_prefix)
self.assertIs(None, node._common_serialised_prefix)
@@ -194,7 +197,8 @@
def assertDeserialiseErrors(self, text):
self.assertRaises((ValueError, IndexError),
- self.module._deserialise_internal_node, text, 'not-a-real-sha')
+ self.module._deserialise_internal_node, text,
+ stuple('not-a-real-sha',))
def test_raises_on_non_internal(self):
self.assertDeserialiseErrors('')
@@ -211,7 +215,7 @@
def test_deserialise_one(self):
node = self.module._deserialise_internal_node(
- "chknode:\n10\n1\n1\n\na\x00sha1:abcd\n", ('sha1:1234',))
+ "chknode:\n10\n1\n1\n\na\x00sha1:abcd\n", stuple('sha1:1234',))
self.assertIsInstance(node, chk_map.InternalNode)
self.assertEqual(1, len(node))
self.assertEqual(10, node.maximum_size)
@@ -221,7 +225,7 @@
def test_deserialise_with_prefix(self):
node = self.module._deserialise_internal_node(
- "chknode:\n10\n1\n1\npref\na\x00sha1:abcd\n", ('sha1:1234',))
+ "chknode:\n10\n1\n1\npref\na\x00sha1:abcd\n", stuple('sha1:1234',))
self.assertIsInstance(node, chk_map.InternalNode)
self.assertEqual(1, len(node))
self.assertEqual(10, node.maximum_size)
@@ -230,7 +234,7 @@
self.assertEqual({'prefa': ('sha1:abcd',)}, node._items)
node = self.module._deserialise_internal_node(
- "chknode:\n10\n1\n1\npref\n\x00sha1:abcd\n", ('sha1:1234',))
+ "chknode:\n10\n1\n1\npref\n\x00sha1:abcd\n", stuple('sha1:1234',))
self.assertIsInstance(node, chk_map.InternalNode)
self.assertEqual(1, len(node))
self.assertEqual(10, node.maximum_size)
@@ -240,7 +244,8 @@
def test_deserialise_pref_with_null(self):
node = self.module._deserialise_internal_node(
- "chknode:\n10\n1\n1\npref\x00fo\n\x00sha1:abcd\n", ('sha1:1234',))
+ "chknode:\n10\n1\n1\npref\x00fo\n\x00sha1:abcd\n",
+ stuple('sha1:1234',))
self.assertIsInstance(node, chk_map.InternalNode)
self.assertEqual(1, len(node))
self.assertEqual(10, node.maximum_size)
@@ -250,7 +255,8 @@
def test_deserialise_with_null_pref(self):
node = self.module._deserialise_internal_node(
- "chknode:\n10\n1\n1\npref\x00fo\n\x00\x00sha1:abcd\n", ('sha1:1234',))
+ "chknode:\n10\n1\n1\npref\x00fo\n\x00\x00sha1:abcd\n",
+ stuple('sha1:1234',))
self.assertIsInstance(node, chk_map.InternalNode)
self.assertEqual(1, len(node))
self.assertEqual(10, node.maximum_size)
=== modified file 'bzrlib/tests/test_chk_map.py'
--- a/bzrlib/tests/test_chk_map.py 2009-10-08 16:03:45 +0000
+++ b/bzrlib/tests/test_chk_map.py 2009-10-20 22:13:23 +0000
@@ -31,8 +31,8 @@
LeafNode,
Node,
)
+from bzrlib.static_tuple import StaticTuple
-key_types = (tuple, chk_map._key_type)
class TestNode(tests.TestCase):
@@ -74,6 +74,7 @@
search_key_func=None):
if chk_bytes is None:
chk_bytes = self.get_chk_bytes()
+ a_dict = dict((StaticTuple(*k), v) for k, v in a_dict.iteritems())
root_key = CHKMap.from_dict(chk_bytes, a_dict,
maximum_size=maximum_size, key_width=key_width,
search_key_func=search_key_func)
@@ -832,13 +833,13 @@
# 'ab' and 'ac' nodes
chkmap.map(('aad',), 'v')
self.assertIsInstance(chkmap._root_node._items['aa'], InternalNode)
- self.assertIsInstance(chkmap._root_node._items['ab'], key_types)
- self.assertIsInstance(chkmap._root_node._items['ac'], key_types)
+ self.assertIsInstance(chkmap._root_node._items['ab'], StaticTuple)
+ self.assertIsInstance(chkmap._root_node._items['ac'], StaticTuple)
# Unmapping 'acd' can notice that 'aa' is an InternalNode and not have
# to map in 'ab'
chkmap.unmap(('acd',))
self.assertIsInstance(chkmap._root_node._items['aa'], InternalNode)
- self.assertIsInstance(chkmap._root_node._items['ab'], key_types)
+ self.assertIsInstance(chkmap._root_node._items['ab'], StaticTuple)
def test_unmap_without_fitting_doesnt_page_in(self):
store = self.get_chk_bytes()
@@ -861,8 +862,8 @@
chkmap.map(('aaf',), 'v')
# At this point, the previous nodes should not be paged in, but the
# newly added nodes would be
- self.assertIsInstance(chkmap._root_node._items['aaa'], key_types)
- self.assertIsInstance(chkmap._root_node._items['aab'], key_types)
+ self.assertIsInstance(chkmap._root_node._items['aaa'], StaticTuple)
+ self.assertIsInstance(chkmap._root_node._items['aab'], StaticTuple)
self.assertIsInstance(chkmap._root_node._items['aac'], LeafNode)
self.assertIsInstance(chkmap._root_node._items['aad'], LeafNode)
self.assertIsInstance(chkmap._root_node._items['aae'], LeafNode)
@@ -870,8 +871,8 @@
# Now unmapping one of the new nodes will use only the already-paged-in
# nodes to determine that we don't need to do more.
chkmap.unmap(('aaf',))
- self.assertIsInstance(chkmap._root_node._items['aaa'], key_types)
- self.assertIsInstance(chkmap._root_node._items['aab'], key_types)
+ self.assertIsInstance(chkmap._root_node._items['aaa'], StaticTuple)
+ self.assertIsInstance(chkmap._root_node._items['aab'], StaticTuple)
self.assertIsInstance(chkmap._root_node._items['aac'], LeafNode)
self.assertIsInstance(chkmap._root_node._items['aad'], LeafNode)
self.assertIsInstance(chkmap._root_node._items['aae'], LeafNode)
@@ -898,9 +899,9 @@
chkmap.map(('aad',), 'v')
# At this point, the previous nodes should not be paged in, but the
# newly added node would be
- self.assertIsInstance(chkmap._root_node._items['aaa'], key_types)
- self.assertIsInstance(chkmap._root_node._items['aab'], key_types)
- self.assertIsInstance(chkmap._root_node._items['aac'], key_types)
+ self.assertIsInstance(chkmap._root_node._items['aaa'], StaticTuple)
+ self.assertIsInstance(chkmap._root_node._items['aab'], StaticTuple)
+ self.assertIsInstance(chkmap._root_node._items['aac'], StaticTuple)
self.assertIsInstance(chkmap._root_node._items['aad'], LeafNode)
# Unmapping the new node will check the existing nodes to see if they
# would fit.
@@ -938,9 +939,9 @@
chkmap.map(('aad',), 'v')
# At this point, the previous nodes should not be paged in, but the
# newly added node would be
- self.assertIsInstance(chkmap._root_node._items['aaa'], key_types)
- self.assertIsInstance(chkmap._root_node._items['aab'], key_types)
- self.assertIsInstance(chkmap._root_node._items['aac'], key_types)
+ self.assertIsInstance(chkmap._root_node._items['aaa'], StaticTuple)
+ self.assertIsInstance(chkmap._root_node._items['aab'], StaticTuple)
+ self.assertIsInstance(chkmap._root_node._items['aac'], StaticTuple)
self.assertIsInstance(chkmap._root_node._items['aad'], LeafNode)
# Now clear the page cache, and only include 2 of the children in the
# cache
@@ -955,7 +956,7 @@
# Unmapping the new node will check the nodes from the page cache
# first, and not have to read in 'aaa'
chkmap.unmap(('aad',))
- self.assertIsInstance(chkmap._root_node._items['aaa'], key_types)
+ self.assertIsInstance(chkmap._root_node._items['aaa'], StaticTuple)
self.assertIsInstance(chkmap._root_node._items['aab'], LeafNode)
self.assertIsInstance(chkmap._root_node._items['aac'], LeafNode)
@@ -975,9 +976,9 @@
chkmap.map(('aaf',), 'val')
# At this point, the previous nodes should not be paged in, but the
# newly added node would be
- self.assertIsInstance(chkmap._root_node._items['aaa'], key_types)
- self.assertIsInstance(chkmap._root_node._items['aab'], key_types)
- self.assertIsInstance(chkmap._root_node._items['aac'], key_types)
+ self.assertIsInstance(chkmap._root_node._items['aaa'], StaticTuple)
+ self.assertIsInstance(chkmap._root_node._items['aab'], StaticTuple)
+ self.assertIsInstance(chkmap._root_node._items['aac'], StaticTuple)
self.assertIsInstance(chkmap._root_node._items['aad'], LeafNode)
self.assertIsInstance(chkmap._root_node._items['aae'], LeafNode)
self.assertIsInstance(chkmap._root_node._items['aaf'], LeafNode)
@@ -985,9 +986,9 @@
# Unmapping a new node will see the other nodes that are already in
# memory, and not need to page in anything else
chkmap.unmap(('aad',))
- self.assertIsInstance(chkmap._root_node._items['aaa'], key_types)
- self.assertIsInstance(chkmap._root_node._items['aab'], key_types)
- self.assertIsInstance(chkmap._root_node._items['aac'], key_types)
+ self.assertIsInstance(chkmap._root_node._items['aaa'], StaticTuple)
+ self.assertIsInstance(chkmap._root_node._items['aab'], StaticTuple)
+ self.assertIsInstance(chkmap._root_node._items['aac'], StaticTuple)
self.assertIsInstance(chkmap._root_node._items['aae'], LeafNode)
self.assertIsInstance(chkmap._root_node._items['aaf'], LeafNode)
@@ -1032,8 +1033,8 @@
{('a',): 'content here', ('b',): 'more content'},
chk_bytes=basis._store, maximum_size=10)
list(target.iter_changes(basis))
- self.assertIsInstance(target._root_node, key_types)
- self.assertIsInstance(basis._root_node, key_types)
+ self.assertIsInstance(target._root_node, StaticTuple)
+ self.assertIsInstance(basis._root_node, StaticTuple)
def test_iter_changes_ab_ab_changed_values_shown(self):
basis = self._get_map({('a',): 'content here', ('b',): 'more content'},
@@ -1132,7 +1133,7 @@
def test_iteritems_selected_one_of_two_items(self):
chkmap = self._get_map( {("a",):"content here", ("b",):"more content"})
self.assertEqual({("a",): "content here"},
- self.to_dict(chkmap, [("a",)]))
+ self.to_dict(chkmap, [StaticTuple("a",)]))
def test_iteritems_keys_prefixed_by_2_width_nodes(self):
chkmap = self._get_map(
@@ -1141,20 +1142,23 @@
maximum_size=10, key_width=2)
self.assertEqual(
{("a", "a"): "content here", ("a", "b"): 'more content'},
- self.to_dict(chkmap, [("a",)]))
+ self.to_dict(chkmap, [StaticTuple("a",)]))
def test_iteritems_keys_prefixed_by_2_width_nodes_hashed(self):
search_key_func = chk_map.search_key_registry.get('hash-16-way')
- self.assertEqual('E8B7BE43\x00E8B7BE43', search_key_func(('a', 'a')))
- self.assertEqual('E8B7BE43\x0071BEEFF9', search_key_func(('a', 'b')))
- self.assertEqual('71BEEFF9\x0000000000', search_key_func(('b', '')))
+ self.assertEqual('E8B7BE43\x00E8B7BE43',
+ search_key_func(StaticTuple('a', 'a')))
+ self.assertEqual('E8B7BE43\x0071BEEFF9',
+ search_key_func(StaticTuple('a', 'b')))
+ self.assertEqual('71BEEFF9\x0000000000',
+ search_key_func(StaticTuple('b', '')))
chkmap = self._get_map(
{("a","a"):"content here", ("a", "b",):"more content",
("b", ""): 'boring content'},
maximum_size=10, key_width=2, search_key_func=search_key_func)
self.assertEqual(
{("a", "a"): "content here", ("a", "b"): 'more content'},
- self.to_dict(chkmap, [("a",)]))
+ self.to_dict(chkmap, [StaticTuple("a",)]))
def test_iteritems_keys_prefixed_by_2_width_one_leaf(self):
chkmap = self._get_map(
@@ -1162,7 +1166,7 @@
("b", ""): 'boring content'}, key_width=2)
self.assertEqual(
{("a", "a"): "content here", ("a", "b"): 'more content'},
- self.to_dict(chkmap, [("a",)]))
+ self.to_dict(chkmap, [StaticTuple("a",)]))
def test___len__empty(self):
chkmap = self._get_map({})
@@ -1377,9 +1381,9 @@
chkmap = chk_map.CHKMap(chk_bytes, None,
search_key_func=chk_map._search_key_16)
chkmap._root_node.set_maximum_size(10)
- chkmap.map(('1',), 'foo')
- chkmap.map(('2',), 'bar')
- chkmap.map(('3',), 'baz')
+ chkmap.map(StaticTuple('1',), 'foo')
+ chkmap.map(StaticTuple('2',), 'bar')
+ chkmap.map(StaticTuple('3',), 'baz')
self.assertEqualDiff("'' InternalNode\n"
" '1' LeafNode\n"
" ('2',) 'bar'\n"
@@ -1393,7 +1397,7 @@
search_key_func=chk_map._search_key_16)
# We can get the values back correctly
self.assertEqual([(('1',), 'foo')],
- list(chkmap.iteritems([('1',)])))
+ list(chkmap.iteritems([StaticTuple('1',)])))
self.assertEqualDiff("'' InternalNode\n"
" '1' LeafNode\n"
" ('2',) 'bar'\n"
@@ -1408,9 +1412,9 @@
chkmap = chk_map.CHKMap(chk_bytes, None,
search_key_func=chk_map._search_key_255)
chkmap._root_node.set_maximum_size(10)
- chkmap.map(('1',), 'foo')
- chkmap.map(('2',), 'bar')
- chkmap.map(('3',), 'baz')
+ chkmap.map(StaticTuple('1',), 'foo')
+ chkmap.map(StaticTuple('2',), 'bar')
+ chkmap.map(StaticTuple('3',), 'baz')
self.assertEqualDiff("'' InternalNode\n"
" '\\x1a' LeafNode\n"
" ('2',) 'bar'\n"
@@ -1424,7 +1428,7 @@
search_key_func=chk_map._search_key_255)
# We can get the values back correctly
self.assertEqual([(('1',), 'foo')],
- list(chkmap.iteritems([('1',)])))
+ list(chkmap.iteritems([StaticTuple('1',)])))
self.assertEqualDiff("'' InternalNode\n"
" '\\x1a' LeafNode\n"
" ('2',) 'bar'\n"
@@ -1450,41 +1454,6 @@
, chkmap._dump_tree())
-class TestSearchKeyFuncs(tests.TestCase):
-
- def assertSearchKey16(self, expected, key):
- self.assertEqual(expected, chk_map._search_key_16(key))
-
- def assertSearchKey255(self, expected, key):
- actual = chk_map._search_key_255(key)
- self.assertEqual(expected, actual, 'actual: %r' % (actual,))
-
- def test_simple_16(self):
- self.assertSearchKey16('8C736521', ('foo',))
- self.assertSearchKey16('8C736521\x008C736521', ('foo', 'foo'))
- self.assertSearchKey16('8C736521\x0076FF8CAA', ('foo', 'bar'))
- self.assertSearchKey16('ED82CD11', ('abcd',))
-
- def test_simple_255(self):
- self.assertSearchKey255('\x8cse!', ('foo',))
- self.assertSearchKey255('\x8cse!\x00\x8cse!', ('foo', 'foo'))
- self.assertSearchKey255('\x8cse!\x00v\xff\x8c\xaa', ('foo', 'bar'))
- # The standard mapping for these would include '\n', so it should be
- # mapped to '_'
- self.assertSearchKey255('\xfdm\x93_\x00P_\x1bL', ('<', 'V'))
-
- def test_255_does_not_include_newline(self):
- # When mapping via _search_key_255, we should never have the '\n'
- # character, but all other 255 values should be present
- chars_used = set()
- for char_in in range(256):
- search_key = chk_map._search_key_255((chr(char_in),))
- chars_used.update(search_key)
- all_chars = set([chr(x) for x in range(256)])
- unused_chars = all_chars.symmetric_difference(chars_used)
- self.assertEqual(set('\n'), unused_chars)
-
-
class TestLeafNode(TestCaseWithStore):
def test_current_size_empty(self):
@@ -1909,18 +1878,18 @@
search_key_func = chk_map.search_key_registry.get('hash-255-way')
node = InternalNode(search_key_func=search_key_func)
leaf1 = LeafNode(search_key_func=search_key_func)
- leaf1.map(None, ('foo bar',), 'quux')
+ leaf1.map(None, StaticTuple('foo bar',), 'quux')
leaf2 = LeafNode(search_key_func=search_key_func)
- leaf2.map(None, ('strange',), 'beast')
- self.assertEqual('\xbeF\x014', search_key_func(('foo bar',)))
- self.assertEqual('\x85\xfa\xf7K', search_key_func(('strange',)))
+ leaf2.map(None, StaticTuple('strange',), 'beast')
+ self.assertEqual('\xbeF\x014', search_key_func(StaticTuple('foo bar',)))
+ self.assertEqual('\x85\xfa\xf7K', search_key_func(StaticTuple('strange',)))
node.add_node("\xbe", leaf1)
# This sets up a path that should not be followed - it will error if
# the code tries to.
node._items['\xbe'] = None
node.add_node("\x85", leaf2)
self.assertEqual([(('strange',), 'beast')],
- sorted(node.iteritems(None, [('strange',), ('weird',)])))
+ sorted(node.iteritems(None, [StaticTuple('strange',), StaticTuple('weird',)])))
def test_iteritems_partial_empty(self):
node = InternalNode()
@@ -1933,7 +1902,7 @@
# Ensure test validity: nothing paged in below the root.
self.assertEqual(2,
len([value for value in node._items.values()
- if type(value) in key_types]))
+ if type(value) is StaticTuple]))
# now, mapping to k3 should add a k3 leaf
prefix, nodes = node.map(None, ('k3',), 'quux')
self.assertEqual("k", prefix)
@@ -1972,7 +1941,7 @@
# Ensure test validity: nothing paged in below the root.
self.assertEqual(2,
len([value for value in node._items.values()
- if type(value) in key_types]))
+ if type(value) is StaticTuple]))
# now, mapping to k23 causes k22 ('k2' in node) to split into k22 and
# k23, which for simplicity in the current implementation generates
# a new internal node between node, and k22/k23.
@@ -2017,9 +1986,12 @@
node = InternalNode(search_key_func=search_key_func)
node._key_width = 2
node._node_width = 4
- self.assertEqual('E8B7BE43\x0071BEEFF9', search_key_func(('a', 'b')))
- self.assertEqual('E8B7', node._search_prefix_filter(('a', 'b')))
- self.assertEqual('E8B7', node._search_prefix_filter(('a',)))
+ self.assertEqual('E8B7BE43\x0071BEEFF9', search_key_func(
+ StaticTuple('a', 'b')))
+ self.assertEqual('E8B7', node._search_prefix_filter(
+ StaticTuple('a', 'b')))
+ self.assertEqual('E8B7', node._search_prefix_filter(
+ StaticTuple('a',)))
def test_unmap_k23_from_k1_k22_k23_gives_k1_k22_tree_new(self):
chkmap = self._get_map(
@@ -2159,7 +2131,7 @@
def help__read_all_roots(self, search_key_func):
c_map = self.make_root_only_map(search_key_func=search_key_func)
key1 = c_map.key()
- c_map.map(('aaa',), 'new aaa content')
+ c_map.map(StaticTuple('aaa',), 'new aaa content')
key2 = c_map._save()
diff = self.get_difference([key2], [key1], search_key_func)
root_results = [record.key for record in diff._read_all_roots()]
More information about the bazaar-commits
mailing list