Rev 75: Update SQLiteExpandedDatabase so that it includes all nested documents as well. in http://bazaar.launchpad.net/+branch/u1db
John Arbash Meinel
john at arbash-meinel.com
Fri Oct 14 09:49:52 UTC 2011
At http://bazaar.launchpad.net/+branch/u1db
------------------------------------------------------------
revno: 75 [merge]
revision-id: john at arbash-meinel.com-20111014094922-hqsqy2rpdwr5l18m
parent: john at arbash-meinel.com-20111014091356-b9031b3gk4njjpac
parent: john at arbash-meinel.com-20111014094854-fe5lm7rkyolcn6jg
committer: John Arbash Meinel <john at arbash-meinel.com>
branch nick: u1db
timestamp: Fri 2011-10-14 11:49:22 +0200
message:
Update SQLiteExpandedDatabase so that it includes all nested documents as well.
modified:
u1db/backends/sqlite_backend.py sqlite_backend.py-20110908122026-u745g3ftpndrgjl0-1
u1db/tests/test_backends.py test_client.py-20110907100834-dekceojbjm2ken0c-6
u1db/tests/test_sqlite_backend.py test_sqlite_backend.-20110908122026-u745g3ftpndrgjl0-2
-------------- next part --------------
=== modified file 'u1db/backends/sqlite_backend.py'
--- a/u1db/backends/sqlite_backend.py 2011-10-14 09:13:34 +0000
+++ b/u1db/backends/sqlite_backend.py 2011-10-14 09:48:54 +0000
@@ -174,6 +174,32 @@
self._put_and_update_indexes(doc_id, old_doc, new_rev, doc)
return new_rev
+ def _expand_to_fields(self, doc_id, base_field, raw_doc):
+ """Convert a dict representation into named fields.
+
+ So something like: {'key1': 'val1', 'key2': 'val2'}
+ gets converted into: [(doc_id, 'key1', 'val1', 0)
+ (doc_id, 'key2', 'val2', 0)]
+ :param doc_id: Just added to every record.
+ :param base_field: if set, these are nested keys, so each field should
+ be appropriately prefixed.
+ :param raw_doc: The python dictionary.
+ """
+ # TODO: Handle lists
+ values = []
+ for field_name, value in raw_doc.iteritems():
+ if base_field:
+ full_name = base_field + '.' + field_name
+ else:
+ full_name = field_name
+ if value is None or isinstance(value, (int, float, basestring)):
+ values.append((doc_id, full_name, value, len(values)))
+ else:
+ subvalues = self._expand_to_fields(doc_id, full_name, value)
+ for _, subfield_name, val, _ in subvalues:
+ values.append((doc_id, subfield_name, val, len(values)))
+ return values
+
def _put_and_update_indexes(self, doc_id, old_doc, new_rev, doc):
"""Actually insert a document into the database.
@@ -366,9 +392,9 @@
else:
c.execute("INSERT INTO document VALUES (?, ?, ?)",
(doc_id, new_rev, doc))
- values = [(doc_id, field_name, value) for field_name, value in
- raw_doc.iteritems()
- if isinstance(value, (int, float, basestring))]
+ values = self._expand_to_fields(doc_id, None, raw_doc)
+ # Strip off the 'offset' column.
+ values = [x[:3] for x in values]
c.executemany("INSERT INTO document_fields VALUES (?, ?, ?)",
values)
c.execute("INSERT INTO transaction_log(doc_id) VALUES (?)",
@@ -403,6 +429,7 @@
if indexed_fields:
# It is expected that len(indexed_fields) is shorter than
# len(raw_doc)
+ # TODO: Handle nested indexed fields.
values = [(doc_id, field_name, raw_doc[field_name])
for field_name in indexed_fields
if field_name in raw_doc]
@@ -455,32 +482,6 @@
def _extra_schema_init(self, c):
c.execute("ALTER TABLE document_fields ADD COLUMN offset INT")
- def _convert_to_fields(self, doc_id, base_field, raw_doc):
- """Convert a dict representation into named fields.
-
- So something like: {'key1': 'val1', 'key2': 'val2'}
- gets converted into: [(doc_id, 'key1', 'val1', 0)
- (doc_id, 'key2', 'val2', 0)]
- :param doc_id: Just added to every record.
- :param base_field: if set, these are nested keys, so each field should
- be appropriately prefixed.
- :param raw_doc: The python dictionary.
- """
- # TODO: Handle lists
- values = []
- for field_name, value in raw_doc.iteritems():
- if base_field:
- full_name = base_field + '.' + field_name
- else:
- full_name = field_name
- if isinstance(value, (int, float, basestring)):
- values.append((doc_id, full_name, value, len(values)))
- else:
- subvalues = self._convert_to_fields(doc_id, full_name, value)
- for _, subfield_name, val, _ in subvalues:
- values.append((doc_id, subfield_name, val, len(values)))
- return values
-
def _put_and_update_indexes(self, doc_id, old_doc, new_rev, doc):
c = self._db_handle.cursor()
if doc:
@@ -498,7 +499,7 @@
else:
c.execute("INSERT INTO document VALUES (?, ?, ?)",
(doc_id, new_rev, doc_content))
- values = self._convert_to_fields(doc_id, None, raw_doc)
+ values = self._expand_to_fields(doc_id, None, raw_doc)
c.executemany("INSERT INTO document_fields VALUES (?, ?, ?, ?)",
values)
c.execute("INSERT INTO transaction_log(doc_id) VALUES (?)",
=== modified file 'u1db/tests/test_backends.py'
--- a/u1db/tests/test_backends.py 2011-10-13 14:16:06 +0000
+++ b/u1db/tests/test_backends.py 2011-10-14 09:48:54 +0000
@@ -171,6 +171,11 @@
doc_id, new_rev = self.c.create_doc(nested_doc)
self.assertEqual((new_rev, nested_doc, False), self.c.get_doc(doc_id))
+ def test_handles_doc_with_null(self):
+ doc_id, new_rev = self.c.create_doc('{"key": null}')
+ self.assertEqual((new_rev, '{"key": null}', False),
+ self.c.get_doc(doc_id))
+
def test__get_sync_info(self):
self.assertEqual(('test', 0, 0), self.c._get_sync_info('other'))
=== modified file 'u1db/tests/test_sqlite_backend.py'
--- a/u1db/tests/test_sqlite_backend.py 2011-10-14 09:13:34 +0000
+++ b/u1db/tests/test_sqlite_backend.py 2011-10-14 09:48:54 +0000
@@ -24,6 +24,7 @@
simple_doc = '{"key": "value"}'
+nested_doc = '{"key": "value", "sub": {"doc": "underneath"}}'
class TestSQLiteExpandedDatabase(tests.TestCase):
@@ -134,6 +135,15 @@
(doc1_id, "key2", "valy"),
], c.fetchall())
+ def test_put_updates_nested_fields(self):
+ doc1_id, doc1_rev = self.db.create_doc(nested_doc)
+ c = self.db._get_sqlite_handle().cursor()
+ c.execute("SELECT doc_id, field_name, value FROM document_fields"
+ " ORDER BY doc_id, field_name, value")
+ self.assertEqual([(doc1_id, "key", "value"),
+ (doc1_id, "sub.doc", "underneath"),
+ ], c.fetchall())
+
class TestSQLitePartialExpandDatabase(tests.TestCase):
More information about the bazaar-commits
mailing list