[Checkins] SVN: Sandbox/luciano/kirbi/ integration of kirbifetch to
kirbi; review of XML-RPC API for fetching
Luciano Ramalho
luciano at ramalho.org
Sun Aug 5 23:37:21 EDT 2007
Log message for revision 78616:
integration of kirbifetch to kirbi; review of XML-RPC API for fetching
metadata
Changed:
U Sandbox/luciano/kirbi/kirbifetch/src/kirbifetch/fetch.py
U Sandbox/luciano/kirbi/kirbifetch/src/kirbifetch/interfaces.py
U Sandbox/luciano/kirbi/kirbifetch/src/kirbifetch/source_amazon.py
U Sandbox/luciano/kirbi/kirbifetch/src/kirbifetch/tests/dummy_server.py
U Sandbox/luciano/kirbi/src/kirbi/book.py
-=-
Modified: Sandbox/luciano/kirbi/kirbifetch/src/kirbifetch/fetch.py
===================================================================
--- Sandbox/luciano/kirbi/kirbifetch/src/kirbifetch/fetch.py 2007-08-06 02:36:24 UTC (rev 78615)
+++ Sandbox/luciano/kirbi/kirbifetch/src/kirbifetch/fetch.py 2007-08-06 03:37:17 UTC (rev 78616)
@@ -51,7 +51,8 @@
def downloadedItemsPage(self, xml, isbns):
book_list = self.source.parseMultipleBookDetails(xml)
- delay = 1
+ deferred = self.pollServer.callRemote(self.callback, book_list)
+ deferred.addCallback(self.uploaded).addErrback(self.uploadError)
for book in book_list:
url = book.get('image_url')
if url:
@@ -66,7 +67,7 @@
out = file(path.join(self.source.name,filename), 'w')
out.write(xml.replace('><','>\n<'))
out.close()
- pprint(book_list)
+
def downloadedImage(self, bytes, filename):
# XXX: find a proper way to calculate the static image dir
@@ -81,14 +82,17 @@
def downloadError(self, error, url):
print 'Error in deferred download (url=%s): %s' % (url, error)
- def deletedPending(n):
- print 'deleted:', n
+ def uploaded(self, number):
+ print 'books uploaded:', number
+ def uploadError(self, error):
+ print 'Error in deferred upload:', error
+
if __name__ == '__main__':
xmlrpc_url = 'http://localhost:8080/RPC2'
poll_method = 'dump_pending_isbns'
- callback = 'add_book'
+ callback = 'add_books'
fetcher = Fetch(xmlrpc_url, poll_method, callback, amazon.Source())
reactor.callLater(0, fetcher.poll)
print 'reactor start'
Modified: Sandbox/luciano/kirbi/kirbifetch/src/kirbifetch/interfaces.py
===================================================================
--- Sandbox/luciano/kirbi/kirbifetch/src/kirbifetch/interfaces.py 2007-08-06 02:36:24 UTC (rev 78615)
+++ Sandbox/luciano/kirbi/kirbifetch/src/kirbifetch/interfaces.py 2007-08-06 03:37:17 UTC (rev 78616)
@@ -9,7 +9,7 @@
name = DottedName(
title = u"Name to identify source in metadata records",
- required = 1,
+ required = True,
)
max_ids_per_request = Int(
Modified: Sandbox/luciano/kirbi/kirbifetch/src/kirbifetch/source_amazon.py
===================================================================
--- Sandbox/luciano/kirbi/kirbifetch/src/kirbifetch/source_amazon.py 2007-08-06 02:36:24 UTC (rev 78615)
+++ Sandbox/luciano/kirbi/kirbifetch/src/kirbifetch/source_amazon.py 2007-08-06 03:37:17 UTC (rev 78616)
@@ -48,7 +48,7 @@
('edition', 'ItemAttributes/Edition'),
('publisher', 'ItemAttributes/Publisher'),
('issued', 'ItemAttributes/PublicationDate'),
- ('subject', 'ItemAttributes/DeweyDecimalNumber'),
+ ('subjects', 'ItemAttributes/DeweyDecimalNumber'),
('image_url', 'LargeImage/URL'),
('source_url', 'DetailPageURL'),
('source_item_id', 'ASIN'),
@@ -118,7 +118,7 @@
if error_code is None:
book_list = []
for item in root.findall(self.nsPath('Items/Item')):
- book_dic = {'source':self.name}
+ book_dic = {}
for field, tag in FIELD_MAP:
elem = item.find(self.nsPath(tag))
if elem is not None:
@@ -135,6 +135,11 @@
creators.append(creator)
if creators:
book_dic['creators'] = creators
+ if book_dic.get('subjects'):
+ # subjects is a Tuple field
+ book_dic['subjects'] = (book_dic['subjects'],)
+ book_dic['source'] = self.name
+
book_list.append(book_dic)
return book_list
Modified: Sandbox/luciano/kirbi/kirbifetch/src/kirbifetch/tests/dummy_server.py
===================================================================
--- Sandbox/luciano/kirbi/kirbifetch/src/kirbifetch/tests/dummy_server.py 2007-08-06 02:36:24 UTC (rev 78615)
+++ Sandbox/luciano/kirbi/kirbifetch/src/kirbifetch/tests/dummy_server.py 2007-08-06 03:37:17 UTC (rev 78616)
@@ -44,8 +44,9 @@
print 'list%s' % self.pending
return self.pending
- def add_book(self, book):
- pprint(book)
+ def add_books(self, books):
+ pprint(books)
+ return len(books)
if __name__=='__main__':
Modified: Sandbox/luciano/kirbi/src/kirbi/book.py
===================================================================
--- Sandbox/luciano/kirbi/src/kirbi/book.py 2007-08-06 02:36:24 UTC (rev 78615)
+++ Sandbox/luciano/kirbi/src/kirbi/book.py 2007-08-06 03:37:17 UTC (rev 78616)
@@ -40,15 +40,28 @@
)
creators = schema.Tuple(title=u"Authors",
- required=False,
- value_type=schema.TextLine(),
- default=())
+ value_type=schema.TextLine(),
+ default=())
edition = schema.TextLine(title=u"Edition", required=False)
publisher = schema.TextLine(title=u"Publisher", required=False)
issued = schema.TextLine(title=u"Issued", required=False)
# TODO: set a vocabulary for language
language = schema.TextLine(title=u"Language", required=False)
+
+ subjects = schema.Tuple(title=u"Subjects",
+ value_type=schema.TextLine(),
+ default=())
+
+ source = schema.TextLine(title=u"Record source",
+ description=u"Name of the source of this record.")
+ source_url = schema.URI(title=u"Source URL",
+ description=u"URL of the source of this record.")
+ source_item_id = schema.TextLine(title=u"Item ID at Source",
+ description= (u"Product number or other identifier"
+ u" for this item at source.")
+ )
+
@invariant
def titleOrIsbnGiven(book):
if (not book.title or not book.title.strip()) and (not book.isbn):
@@ -85,7 +98,6 @@
"""
-
implements(IBook)
__title = '' # = __main_title + __title_glue + __sub_title
__main_title = '' # title without sub-title
@@ -97,7 +109,8 @@
__language = None
def __init__(self, title='', isbn13=None, creators=None, edition=None,
- publisher=None, issued=None, language=None):
+ publisher=None, issued=None, language=None, subjects=None,
+ source=None, source_url=None, source_item_id=None):
super(Book, self).__init__()
if isbn13:
self.isbn13 = isbn13
@@ -112,6 +125,9 @@
self.publisher = publisher
self.issued = issued
self.language = language
+ self.source = source
+ self.source_url = source_url
+ self.source_item_id = source_item_id
def getTitle(self):
return self.__title
More information about the Checkins
mailing list