""" For writing to 3Store asynchrounus """ # Copyright 2005, 2006 EIAO Consoritum # This program is distributed under the terms of the GNU General # Public License. # # This file is part of the European Internet Accessibility Observatory # (EIAO) # # EIAO is free software; you can redistribute it and/or modify # it under the terms of the GNU General Public License as published by # the Free Software Foundation; either version 2 of the License, or # (at your option) any later version. # # EIAO is distributed in the hope that it will be useful, # but WITHOUT ANY WARRANTY; without even the implied warranty of # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the # GNU General Public License for more details. # # You should have received a copy of the GNU General Public License # along with EIAO; if not, write to the Free Software # Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, # MA 02110-1301 USA #from threading import Thread #import rdfsql import re from pytriplestore import * import time import timeprofile import RDF __author__ = "Morten Goodwin Olsen" __maintainer__ = "Nils Ulltveit-Moe" __version__ = "$Id$" class asynchwriter: """Asynchrounus RDF writer Usage: >>> def somewritefunction(): >>> ... >>> triples = [...] >>> a = asynchwriter(triples,somewritefunction) >>> a.start() """ def writeRDF(self): """ Parsing and writing RDF Returns None """ pre = time.time() self.dt.cursor.execute("""insert into RAWRDF (rdf) values (%s);""",self.rdf) #f = open('/var/log/eiao/time.log','a') #f.write('Writing to Database: ('+str(len(self.rdf))+') : '+str(time.time()-pre)+'\n') #f.close() return None #The following is all deprecated, but is kept in the code in case we prefer to go back to the old way of writing RDF. #Temp start #f = open('/var/log/eiao/allrdf.log','a') #f.write(str(self.rdf)) #f.write('\n') #f.close() import rdfsql model = RDF.Model() pars = RDF.RDFXMLParser() try: pars.parse_string_into_model(model, self.rdf, base_uri="http://www.eiao.net/rdf/2.0#") except Exception, e: #Bug workaround. RedlandError could not be excepted. if str(e.__class__) == 'RDF.RedlandError': raise IllegalRDFError(self.rdf, str(e)) else: raise e stat = RDF.Statement(subject = None, predicate = None, object = None) statements = model.find_statements(stat) #Temp stop statforwriting = [] allstatements = [stat for stat in statements] for statement in allstatements: #for statement in self.allstatements: #print '.', if statement.subject.is_blank(): subject = str(statement.subject._get_blank_identifier()) else: subject = str(statement.subject.uri) predicate = str(statement.predicate.uri) if statement.object.is_blank(): object_type = rdfsql.ObjURI object = str(statement.object._get_blank_identifier()) elif statement.object.is_literal(): object_type = rdfsql.ObjLiteral if type(statement.object.literal_value['string'])==type(u''): object = statement.object.literal_value['string'].encode('utf-8') else: object = str(statement.object.literal_value['string']) else: object = str(statement.object.uri) object_type = rdfsql.ObjURI if type(object)==type(u''): object = object.encode('utf-8') statforwriting.append([subject,predicate,object,object_type]) #statforwriting.append([re.sub("[^\\\\]'","\\'",subject),re.sub("[^\\\\]'","\\'",predicate),re.sub("[^\\\\]'","\\'",object),object_type]) #statforwriting.append([subject.replace("'","\\'"),predicate.replace("'","\\'"),object.replace("'","\\'"),object_type]) #statforwriting.append([subject,predicate,object,object_type]) #self.dt.writetriple(subject,predicate,object,object_type) #dt.writeRDFTriple(subject,predicate,object,object_type) pre = time.time() self.dt.writetriplesfast(statforwriting) #f = open('/var/log/eiao/time.log','a') #f.write('Writing to Database: ('+str(len(statforwriting))+') : '+str(time.time()-pre)+'\n') #f.close() return None #self.join() def __init__ (self,rdf,writeRDFTriple,host,username, password, database, rdfmodel): """Constructor Keyword arguments: allstatements -- All statements to be written writerRDFTriple -- Function for writing RDF triple host -- Host to connect to username -- Username for connection password -- Password for connection database -- Database for connection rdfmodel -- RDF model for connection """ #Thread.__init__(self) self.rdf = rdf #self.allstatements = allstatements self.writeRDFTriple = writeRDFTriple self.dt = PyTripleStore(host=host,user=username, passwd=password, db=database,model=rdfmodel) def run(self): """ Running the RDF writer in a separate thread Returns None""" #Note, this function is not used timeprofile.mark('dbwrite') self.writeRDF() self.dt.dropconnection() #f = open('/var/log/eiao/time.log','a') #f.write('Writing to Database:'+str(timeprofile.elapsed('dbwrite'))+'\n') #f.close()