[Checkins] SVN: zc.FileStorage/dev/s Another attempt at reducing
memory consumption.
Jim Fulton
jim at zope.com
Thu Dec 6 10:49:02 EST 2007
Log message for revision 82162:
Another attempt at reducing memory consumption.
Changed:
U zc.FileStorage/dev/setup.py
A zc.FileStorage/dev/src/zc/FileStorage/_ILBTree.c
U zc.FileStorage/dev/src/zc/FileStorage/__init__.py
-=-
Modified: zc.FileStorage/dev/setup.py
===================================================================
--- zc.FileStorage/dev/setup.py 2007-12-06 15:33:45 UTC (rev 82161)
+++ zc.FileStorage/dev/setup.py 2007-12-06 15:49:00 UTC (rev 82162)
@@ -13,7 +13,11 @@
packages = find_packages('src'),
ext_modules=[
Extension('zc.FileStorage._zc_FileStorage_posix_fadvise',
- ['src/zc/FileStorage/_zc_FileStorage_posix_fadvise.c'])
+ ['src/zc/FileStorage/_zc_FileStorage_posix_fadvise.c']),
+ Extension('zc.FileStorage._ILBTree',
+ ['src/zc/FileStorage/_ILBTree.c'],
+ include_dirs=['3.8/src'],
+ ),
],
namespace_packages = ['zc'],
package_dir = {'': 'src'},
Added: zc.FileStorage/dev/src/zc/FileStorage/_ILBTree.c
===================================================================
--- zc.FileStorage/dev/src/zc/FileStorage/_ILBTree.c (rev 0)
+++ zc.FileStorage/dev/src/zc/FileStorage/_ILBTree.c 2007-12-06 15:49:00 UTC (rev 82162)
@@ -0,0 +1,67 @@
+/*############################################################################
+#
+# Copyright (c) 2004 Zope Corporation and Contributors.
+# All Rights Reserved.
+#
+# This software is subject to the provisions of the Zope Public License,
+# Version 2.1 (ZPL). A copy of the ZPL should accompany this distribution.
+# THIS SOFTWARE IS PROVIDED "AS IS" AND ANY AND ALL EXPRESS OR IMPLIED
+# WARRANTIES ARE DISCLAIMED, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
+# WARRANTIES OF TITLE, MERCHANTABILITY, AGAINST INFRINGEMENT, AND FITNESS
+# FOR A PARTICULAR PURPOSE.
+#
+############################################################################*/
+
+#define MASTER_ID "$Id: _IIBTree.c 25186 2004-06-02 15:07:33Z jim $\n"
+
+/* IIBTree - int key, int value BTree
+
+ Implements a collection using int type keys
+ and int type values
+*/
+
+/* Setup template macros */
+
+#define PERSISTENT
+
+#define MOD_NAME_PREFIX "IL"
+#define INITMODULE init_ILBTree
+#define DEFAULT_MAX_BUCKET_SIZE 120
+#define DEFAULT_MAX_BTREE_SIZE 500
+
+#include "BTrees/intkeymacros.h"
+
+
+#define VALUEMACROS_H "$Id:$\n"
+
+#define NEED_LONG_LONG_SUPPORT
+#define VALUE_TYPE PY_LONG_LONG
+#define VALUE_PARSE "L"
+#define COPY_VALUE_TO_OBJECT(O, K) O=longlong_as_object(K)
+#define COPY_VALUE_FROM_ARG(TARGET, ARG, STATUS) \
+ if (PyInt_Check(ARG)) TARGET=PyInt_AS_LONG(ARG); else \
+ if (longlong_check(ARG)) TARGET=PyLong_AsLongLong(ARG); else \
+ if (PyLong_Check(ARG)) { \
+ PyErr_SetString(PyExc_ValueError, "long integer out of range"); \
+ (STATUS)=0; (TARGET)=0; } \
+ else { \
+ PyErr_SetString(PyExc_TypeError, "expected integer value"); \
+ (STATUS)=0; (TARGET)=0; }
+
+
+#undef VALUE_TYPE_IS_PYOBJECT
+#define TEST_VALUE(K, T) (((K) < (T)) ? -1 : (((K) > (T)) ? 1: 0))
+#define VALUE_SAME(VALUE, TARGET) ( (VALUE) == (TARGET) )
+#define DECLARE_VALUE(NAME) VALUE_TYPE NAME
+#define DECREF_VALUE(k)
+#define INCREF_VALUE(k)
+#define COPY_VALUE(V, E) (V=(E))
+
+#define NORMALIZE_VALUE(V, MIN) ((MIN) > 0) ? ((V)/=(MIN)) : 0
+
+#define MERGE_DEFAULT 1
+#define MERGE(O1, w1, O2, w2) ((O1)*(w1)+(O2)*(w2))
+#define MERGE_WEIGHT(O, w) ((O)*(w))
+
+
+#include "BTrees/BTreeModuleTemplate.c"
Property changes on: zc.FileStorage/dev/src/zc/FileStorage/_ILBTree.c
___________________________________________________________________
Name: svn:eol-style
+ native
Modified: zc.FileStorage/dev/src/zc/FileStorage/__init__.py
===================================================================
--- zc.FileStorage/dev/src/zc/FileStorage/__init__.py 2007-12-06 15:33:45 UTC (rev 82161)
+++ zc.FileStorage/dev/src/zc/FileStorage/__init__.py 2007-12-06 15:49:00 UTC (rev 82162)
@@ -20,7 +20,7 @@
from ZODB.utils import p64, u64, z64
from ZODB.FileStorage.format import TRANS_HDR_LEN
-import BTrees.IOBTree, BTrees.LOBTree
+import BTrees.IOBTree, BTrees.LOBTree, _ILBTree
import ZODB.FileStorage
import ZODB.FileStorage.fspack
import ZODB.fsIndex
@@ -178,12 +178,16 @@
ioid1, ioid2 = divmod(u64(dh.oid), 2147483648L)
ioid2 = int(ioid2)
- references1 = references.get(ioid1)
- if references1 is None:
- references1 = references[ioid1] = BTrees.IOBTree.IOBTree()
+ references_ioid1 = references.get(ioid1)
+ if references_ioid1 is None:
+ references_ioid1 = references[ioid1] = (
+ _ILBTree.ILBTree(), BTrees.IOBTree.IOBTree()
+ )
if merge:
- initial = references1.get(ioid2)
+ initial = references_ioid1[0].get(ioid2)
+ if initial is None:
+ initial = references_ioid1[1].get(ioid2)
else:
initial = None
@@ -198,28 +202,28 @@
else:
refs.add(initial)
if len(refs) == 1:
- refs = refs.pop()
+ references_ioid1[0][ioid2] = refs.pop()
+ references_ioid1[1].pop(ioid2, None)
else:
- refs = tuple(refs)
+ references_ioid1[1][ioid2] = tuple(refs)
+ references_ioid1[0].pop(ioid2, None)
else:
if len(refs) == 1:
- refs = u64(refs.pop())
+ references_ioid1[0][ioid2] = u64(refs.pop())
+ references_ioid1[1].pop(ioid2, None)
else:
refs = set(map(u64, refs))
if len(refs) == 1:
- refs = refs.pop()
+ references_ioid1[0][ioid2] = refs.pop()
+ references_ioid1[1].pop(ioid2, None)
else:
- refs = tuple(refs)
- else:
- refs = initial
- else:
- refs = initial
+ references_ioid1[1][ioid2] = tuple(refs)
+ references_ioid1[0].pop(ioid2, None)
+ return
- if refs is not None:
- references1[ioid2] = refs
- else:
- references1.pop(ioid2, None)
-
+ if (not merge) and references_ioid1[0].pop(ioid2, None) is None:
+ references_ioid1[1].pop(ioid2, None)
+
def gc(self, index, references):
to_do = [0]
reachable = ZODB.fsIndex.fsIndex()
@@ -239,14 +243,17 @@
pass
ioid1, ioid2 = divmod(ioid, 2147483648L)
- references1 = references.get(ioid1)
- if references1:
- refs = references1.pop(int(ioid2), None)
- if refs is not None:
- if refs.__class__ is tuple:
+
+ references_ioid1 = references.get(ioid1)
+ if references_ioid1:
+ ioid2 = int(ioid2)
+ ref = references_ioid1[0].pop(ioid2, None)
+ if ref is not None:
+ to_do.append(ref)
+ else:
+ refs = references_ioid1[1].pop(ioid2, None)
+ if refs:
to_do.extend(refs)
- else:
- to_do.append(refs)
references.clear()
return reachable
More information about the Checkins
mailing list