3 # ljdump.py - livejournal archiver
4 # Greg Hewgill <greg@hewgill.com> http://hewgill.com
9 # This program reads the journal entries from a livejournal (or compatible)
10 # blog site and archives them in a subdirectory named after the journal name.
12 # The configuration is read from "ljdump.config". A sample configuration is
13 # provided in "ljdump.config.sample", which should be copied and then edited.
14 # The configuration settings are:
16 # server - The XMLRPC server URL. This should only need to be changed
17 # if you are dumping a journal that is livejournal-compatible
18 # but is not livejournal itself.
20 # username - The livejournal user name. A subdirectory will be created
21 # with this same name to store the journal entries.
23 # password - The account password. This password is never sent in the
24 # clear; the livejournal "challenge" password mechanism is used.
26 # This program may be run as often as needed to bring the backup copy up
27 # to date. Both new and updated items are downloaded.
31 # This software is provided 'as-is', without any express or implied
32 # warranty. In no event will the author be held liable for any damages
33 # arising from the use of this software.
35 # Permission is granted to anyone to use this software for any purpose,
36 # including commercial applications, and to alter it and redistribute it
37 # freely, subject to the following restrictions:
39 # 1. The origin of this software must not be misrepresented; you must not
40 # claim that you wrote the original software. If you use this software
41 # in a product, an acknowledgment in the product documentation would be
42 # appreciated but is not required.
43 # 2. Altered source versions must be plainly marked as such, and must not be
44 # misrepresented as being the original software.
45 # 3. This notice may not be removed or altered from any source distribution.
47 # Copyright (c) 2005-2006 Greg Hewgill
49 import codecs, md5, os, pickle, pprint, re, sys, urllib2, xml.dom.minidom, xmlrpclib
50 from xml.sax import saxutils
52 def calcchallenge(challenge, password):
53 return md5.new(challenge+md5.new(password).hexdigest()).hexdigest()
55 def flatresponse(response):
58 name = response.readline()
62 name = name[:len(name)-1]
63 value = response.readline()
65 value = value[:len(value)-1]
69 def getljsession(username, password):
70 r = urllib2.urlopen(Server+"/interface/flat", "mode=getchallenge")
71 response = flatresponse(r)
73 r = urllib2.urlopen(Server+"/interface/flat", "mode=sessiongenerate&user=%s&auth_method=challenge&auth_challenge=%s&auth_response=%s" % (username, response['challenge'], calcchallenge(response['challenge'], password)))
74 response = flatresponse(r)
76 return response['ljsession']
78 def dochallenge(params, password):
79 challenge = server.LJ.XMLRPC.getchallenge()
81 'auth_method': "challenge",
82 'auth_challenge': challenge['challenge'],
83 'auth_response': calcchallenge(challenge['challenge'], password)
87 def dumpelement(f, name, e):
88 f.write("<%s>\n" % name)
90 if isinstance(e[k], {}.__class__):
91 dumpelement(f, k, e[k])
93 s = unicode(str(e[k]), "UTF-8")
94 f.write("<%s>%s</%s>\n" % (k, saxutils.escape(s), k))
95 f.write("</%s>\n" % name)
97 def writedump(fn, event):
98 f = codecs.open(fn, "w", "UTF-8")
99 f.write("""<?xml version="1.0"?>\n""")
100 dumpelement(f, "event", event)
103 def createxml(doc, name, map):
104 e = doc.createElement(name)
106 me = doc.createElement(k)
107 me.appendChild(doc.createTextNode(map[k]))
114 return e[0].firstChild.nodeValue
116 config = xml.dom.minidom.parse("ljdump.config")
117 Server = config.documentElement.getElementsByTagName("server")[0].childNodes[0].data
118 Username = config.documentElement.getElementsByTagName("username")[0].childNodes[0].data
119 Password = config.documentElement.getElementsByTagName("password")[0].childNodes[0].data
121 m = re.search("(.*)/interface/xmlrpc", Server)
125 print "Fetching journal entries for: %s" % Username
128 print "Created subdirectory: %s" % Username
132 ljsession = getljsession(Username, Password)
134 server = xmlrpclib.ServerProxy(Server+"/interface/xmlrpc")
143 f = open("%s/.last" % Username, "r")
144 lastsync = f.readline()
145 if lastsync[-1] == '\n':
146 lastsync = lastsync[:len(lastsync)-1]
147 lastmaxid = f.readline()
148 if len(lastmaxid) > 0 and lastmaxid[-1] == '\n':
149 lastmaxid = lastmaxid[:len(lastmaxid)-1]
153 lastmaxid = int(lastmaxid)
157 origlastsync = lastsync
160 r = server.LJ.XMLRPC.syncitems(dochallenge({
161 'username': Username,
163 'lastsync': lastsync,
166 if len(r['syncitems']) == 0:
168 for item in r['syncitems']:
169 if item['item'][0] == 'L':
170 print "Fetching journal entry %s (%s)" % (item['item'], item['action'])
172 e = server.LJ.XMLRPC.getevents(dochallenge({
173 'username': Username,
176 'itemid': item['item'][2:],
178 writedump("%s/%s" % (Username, item['item']), e['events'][0])
180 except xmlrpclib.Fault, x:
181 print "Error getting item: %s" % item['item']
184 lastsync = item['time']
186 # The following code doesn't work because the server rejects our repeated calls.
187 # http://www.livejournal.com/doc/server/ljp.csp.xml-rpc.getevents.html
188 # contains the statement "You should use the syncitems selecttype in
189 # conjuntions [sic] with the syncitems protocol mode", but provides
190 # no other explanation about how these two function calls should
191 # interact. Therefore we just do the above slow one-at-a-time method.
194 # r = server.LJ.XMLRPC.getevents(dochallenge({
195 # 'username': Username,
197 # 'selecttype': "syncitems",
198 # 'lastsync': lastsync,
201 # if len(r['events']) == 0:
203 # for item in r['events']:
204 # writedump("%s/L-%d" % (Username, item['itemid']), item)
206 # lastsync = item['eventtime']
208 print "Fetching journal comments for: %s" % Username
211 f = open("%s/comment.meta" % Username)
212 metacache = pickle.load(f)
218 f = open("%s/user.map" % Username)
219 usermap = pickle.load(f)
226 r = urllib2.urlopen(urllib2.Request(Server+"/export_comments.bml?get=comment_meta&startid=%d" % (maxid+1), headers = {'Cookie': "ljsession="+ljsession}))
227 meta = xml.dom.minidom.parse(r)
229 for c in meta.getElementsByTagName("comment"):
230 id = int(c.getAttribute("id"))
232 'posterid': c.getAttribute("posterid"),
233 'state': c.getAttribute("state"),
237 for u in meta.getElementsByTagName("usermap"):
238 usermap[u.getAttribute("id")] = u.getAttribute("user")
239 if maxid >= int(meta.getElementsByTagName("maxid")[0].firstChild.nodeValue):
242 f = open("%s/comment.meta" % Username, "w")
243 pickle.dump(metacache, f)
246 f = open("%s/user.map" % Username, "w")
247 pickle.dump(usermap, f)
253 r = urllib2.urlopen(urllib2.Request(Server+"/export_comments.bml?get=comment_body&startid=%d" % (maxid+1), headers = {'Cookie': "ljsession="+ljsession}))
254 meta = xml.dom.minidom.parse(r)
256 for c in meta.getElementsByTagName("comment"):
257 id = int(c.getAttribute("id"))
258 jitemid = c.getAttribute("jitemid")
261 'parentid': c.getAttribute("parentid"),
262 'subject': gettext(c.getElementsByTagName("subject")),
263 'date': gettext(c.getElementsByTagName("date")),
264 'body': gettext(c.getElementsByTagName("body")),
265 'state': metacache[id]['state'],
267 if usermap.has_key(c.getAttribute("posterid")):
268 comment["user"] = usermap[c.getAttribute("posterid")]
270 entry = xml.dom.minidom.parse("%s/C-%s" % (Username, jitemid))
272 entry = xml.dom.minidom.getDOMImplementation().createDocument(None, "comments", None)
274 for d in entry.getElementsByTagName("comment"):
275 if int(d.getElementsByTagName("id")[0].firstChild.nodeValue) == id:
279 print "Warning: downloaded duplicate comment id %d in jitemid %s" % (id, jitemid)
281 entry.documentElement.appendChild(createxml(entry, "comment", comment))
282 f = codecs.open("%s/C-%s" % (Username, jitemid), "w", "UTF-8")
288 if maxid >= newmaxid:
293 f = open("%s/.last" % Username, "w")
294 f.write("%s\n" % lastsync)
295 f.write("%s\n" % lastmaxid)
299 print "%d new entries, %d new comments (since %s)" % (newentries, newcomments, origlastsync)
301 print "%d new entries, %d new comments" % (newentries, newcomments)
303 print "%d errors" % errors