Blame
Date:
Thu Nov 1 15:00:58 2012 UTC
Message:
take care of useless args
001
2012-10-21
fredg
#!/usr/bin/env python
002
2012-10-21
fredg
# -*- coding: utf-8 -*-
003
2012-10-21
fredg
#
004
2012-10-21
fredg
# pkgtxt2db.py
005
2012-10-21
fredg
#
006
2012-10-21
fredg
# Copyright 2012 Frédéric Galusik <fredg~at~salixos~dot~org>
007
2012-10-21
fredg
#
008
2012-10-21
fredg
#
009
2012-10-21
fredg
# License: BSD Revised
010
2012-10-21
fredg
#
011
2012-10-21
fredg
# Convert the Slackware/Salix PACKAGES.TXT file to some various database
012
2012-10-21
fredg
# formats : CSV, json, xml
013
2012-10-21
fredg
#
014
2012-10-21
fredg
015
2012-10-21
fredg
016
2012-10-21
fredg
import os
017
2012-10-29
fredg
import sys
018
2012-10-21
fredg
import urllib2
019
2012-10-21
fredg
import gzip
020
2012-10-21
fredg
import re
021
2012-10-29
fredg
import argparse
022
2012-10-21
fredg
023
2012-10-29
fredg
024
2012-10-21
fredg
# Program information
025
2012-10-21
fredg
my_url = 'http://www.salixos.org/wiki/index.php/Pkgtxt2db'
026
2012-10-21
fredg
my_name = 'pkgtxt2db'
027
2012-10-21
fredg
my_version = '0.0'
028
2012-10-21
fredg
029
2012-10-29
fredg
030
2012-10-29
fredg
# Parse the CLI options
031
2012-10-29
fredg
parser = argparse.ArgumentParser(
032
2012-10-30
fredg
prog='pkgtxt2db',
033
2012-10-29
fredg
description='Convert PACKAGES.TXT to DB',
034
2012-10-30
fredg
epilog=
035
2012-11-01
fredg
"i.e. pkgtxt2db -u -t salix --repo \
036
2012-11-01
fredg
x86_64 -r 14.0 -c json -o salix64.json")
037
2012-10-30
fredg
038
2012-10-29
fredg
parser.add_argument('-u', '--update', action="store_true",
039
2012-10-29
fredg
default=False,
040
2012-10-29
fredg
help='Download/update the PACKAGES.TXT file')
041
2012-10-30
fredg
042
2012-10-29
fredg
parser.add_argument('-t', '--target', action="store",
043
2012-11-01
fredg
dest='target',
044
2012-10-29
fredg
help='Choose the O.S.: slackware or salix (default) ')
045
2012-10-30
fredg
046
2012-10-29
fredg
parser.add_argument('--repo', action="store",
047
2012-11-01
fredg
dest='repo',
048
2012-10-29
fredg
help='Choose the arch repo: x86_64 or i486 (default)')
049
2012-10-30
fredg
050
2012-10-29
fredg
parser.add_argument('-e', '--expa', action="store",
051
2012-11-01
fredg
dest='expa',
052
2012-10-29
fredg
help='Choose the slackware extra/patches')
053
2012-10-30
fredg
054
2012-10-29
fredg
parser.add_argument('-r', '--release', action="store",
055
2012-11-01
fredg
dest='release',
056
2012-10-29
fredg
help='Choose the release: 13.0 to 14.0 (default)')
057
2012-10-30
fredg
058
2012-10-29
fredg
parser.add_argument('-c', '--convert', action="store",
059
2012-10-29
fredg
dest='convert',
060
2012-10-29
fredg
help='Choose the DB format: xml, json, csv')
061
2012-10-30
fredg
062
2012-10-29
fredg
parser.add_argument('-o', '--output', action="store",
063
2012-10-29
fredg
dest='output', default='packages',
064
2012-10-29
fredg
help='Choose the name of your DB file')
065
2012-10-30
fredg
066
2012-10-30
fredg
parser.add_argument('--version', action='version',
067
2012-10-30
fredg
version='%(prog)s ' + my_version)
068
2012-10-30
fredg
069
2012-10-30
fredg
if len(sys.argv) == 1:
070
2012-10-30
fredg
sys.exit('Wrong usage, see pkgtxt2db --help')
071
2012-10-30
fredg
072
2012-10-29
fredg
args = parser.parse_args()
073
2012-10-29
fredg
074
2012-10-21
fredg
# vars
075
2012-10-21
fredg
mirror = 'http://salix.enialis.net/'
076
2012-10-21
fredg
pkgtxtz = 'PACKAGES.TXT.gz'
077
2012-10-21
fredg
pkgtxt = 'PACKAGES.TXT'
078
2012-10-29
fredg
update = args.update
079
2012-10-29
fredg
target = args.target
080
2012-10-29
fredg
repo = args.repo
081
2012-10-29
fredg
release = args.release
082
2012-10-29
fredg
expa = args.expa
083
2012-10-29
fredg
convert = args.convert
084
2012-10-29
fredg
output = args.output
085
2012-10-29
fredg
outputfile = '.'.join([output, convert])
086
2012-10-21
fredg
087
2012-10-29
fredg
088
2012-11-01
fredg
# initialise the pkg dictionnary with empty values
089
2012-11-01
fredg
fields = ['name',
090
2012-11-01
fredg
'version',
091
2012-11-01
fredg
'arch',
092
2012-11-01
fredg
'release'
093
2012-11-01
fredg
'location',
094
2012-11-01
fredg
'deps',
095
2012-11-01
fredg
'sizec',
096
2012-11-01
fredg
'sizeu',
097
2012-11-01
fredg
'slackdesc']
098
2012-11-01
fredg
099
2012-11-01
fredg
100
2012-10-21
fredg
def new_pkgdct():
101
2012-10-21
fredg
return dict(zip(fields, ['', ] * len(fields)))
102
2012-10-21
fredg
103
2012-10-21
fredg
104
2012-10-21
fredg
# Fetch PACKAGES.TXT
105
2012-10-30
fredg
def pkgtxturl(repo='i486', target='salix', release='14.0', expa='/'):
106
2012-10-21
fredg
"""
107
2012-10-29
fredg
Download the slackware/salix PACKAGES.TXT.gz from a built URL and unzip it.
108
2012-10-29
fredg
pkgtxturl(repo, target, release, |extra|patches)
109
2012-10-29
fredg
"""
110
2012-10-30
fredg
111
2012-10-30
fredg
# Check the repo
112
2012-10-30
fredg
Rep = {'i486', 'x86_64'}
113
2012-10-30
fredg
if repo not in Rep:
114
2012-10-30
fredg
sys.exit('Choose a valid arch please, aborting.')
115
2012-10-30
fredg
116
2012-10-30
fredg
# Check if the choosen release is a valid one.
117
2012-10-30
fredg
R = {'13.0', '13.1', '13.37', '14.0', 'current'}
118
2012-10-30
fredg
if release not in R:
119
2012-10-30
fredg
sys.exit('Choose a valid release please, aborting.')
120
2012-10-30
fredg
121
2012-10-30
fredg
# Check the target
122
2012-10-29
fredg
if target == 'slackware':
123
2012-10-29
fredg
target = 'slackware-'
124
2012-10-29
fredg
elif target == 'salix':
125
2012-10-29
fredg
target = ''
126
2012-10-29
fredg
else:
127
2012-10-29
fredg
sys.exit('Choose a valid target, aborting.')
128
2012-10-29
fredg
129
2012-10-30
fredg
#
130
2012-10-29
fredg
if expa == 'extra':
131
2012-10-29
fredg
expa = '/extra/'
132
2012-10-29
fredg
elif expa == 'patches':
133
2012-10-29
fredg
expa = '/patches/'
134
2012-10-29
fredg
135
2012-10-30
fredg
slash = '/'
136
2012-10-30
fredg
137
2012-10-30
fredg
# Build the URL to fetch PACKAGES.TXT
138
2012-10-29
fredg
url = mirror + repo + slash + target + release + expa + pkgtxtz
139
2012-10-29
fredg
140
2012-10-21
fredg
# remove old files
141
2012-10-21
fredg
if os.path.isfile(pkgtxtz):
142
2012-10-21
fredg
os.remove(pkgtxtz)
143
2012-10-21
fredg
print "Remove old ", pkgtxtz
144
2012-10-21
fredg
if os.path.isfile(pkgtxt):
145
2012-10-21
fredg
os.remove(pkgtxt)
146
2012-10-21
fredg
print "Remove old ", pkgtxt
147
2012-10-21
fredg
try:
148
2012-10-21
fredg
f = urllib2.urlopen(url)
149
2012-10-21
fredg
print "Fetching ", url
150
2012-10-21
fredg
print ""
151
2012-10-21
fredg
# Open local_file for writing
152
2012-10-21
fredg
with open(os.path.basename(url), "wb") as local_file:
153
2012-10-21
fredg
local_file.write(f.read())
154
2012-10-21
fredg
except urllib2.HTTPError, e:
155
2012-10-21
fredg
print "HTTP Error:", e.code, url
156
2012-10-21
fredg
return False
157
2012-10-21
fredg
except urllib2.URLError, e:
158
2012-10-21
fredg
print "URL Error:", e.reason, url
159
2012-10-21
fredg
return False
160
2012-10-30
fredg
161
2012-10-21
fredg
# unzip it
162
2012-10-21
fredg
fout = open(pkgtxt, 'w')
163
2012-10-21
fredg
with gzip.open(pkgtxtz, 'rb') as f:
164
2012-10-21
fredg
for line in f:
165
2012-10-21
fredg
fout.write(line)
166
2012-10-21
fredg
fout.close()
167
2012-10-21
fredg
168
2012-10-21
fredg
169
2012-10-21
fredg
# to CSV DB
170
2012-10-21
fredg
def tocsv(pkgDct, sep=";"):
171
2012-10-21
fredg
"""
172
2012-10-21
fredg
Export PACKAGES.TXT to a CSV database format.
173
2012-10-21
fredg
The separated string can be choosen with the sep var, default is ;
174
2012-10-21
fredg
"""
175
2012-10-29
fredg
with open(outputfile, 'a') as csvf:
176
2012-10-21
fredg
csvf.write(
177
2012-10-21
fredg
sep.join(map(lambda field: pkgDct.get(field, ''), fields)) + '\n')
178
2012-10-21
fredg
179
2012-10-21
fredg
180
2012-10-21
fredg
# to JSON DB
181
2012-10-21
fredg
def tojson(pkgDct):
182
2012-10-21
fredg
"""
183
2012-10-21
fredg
Export PACKAGES.TXT to a JSON database format
184
2012-10-21
fredg
"""
185
2012-10-22
fredg
with open("pre.json", 'a') as j:
186
2012-10-22
fredg
j.write(' {\n')
187
2012-10-22
fredg
j.write(' \"name\": \"' + pkgDct.get("name") + '\",\n')
188
2012-10-22
fredg
j.write(' \"version\": \"' + pkgDct.get("version") + '\",\n')
189
2012-10-22
fredg
j.write(' \"arch\": \"' + pkgDct.get("arch") + '\",\n')
190
2012-10-22
fredg
j.write(' \"release\": \"' + pkgDct.get("release") + '\",\n')
191
2012-10-22
fredg
j.write(' \"location\": \"' + pkgDct.get("location") + '\",\n')
192
2012-10-22
fredg
j.write(' \"deps\": \"' + pkgDct.get("deps") + '\",\n')
193
2012-10-22
fredg
j.write(' \"sizec\": \"' + pkgDct.get("sizec") + '\",\n')
194
2012-10-22
fredg
j.write(' \"sizeu\": \"' + pkgDct.get("sizeu") + '\",\n')
195
2012-10-22
fredg
j.write(' \"slackdesc\": \"' + pkgDct.get("slackdesc") + '\"\n')
196
2012-10-22
fredg
j.write(' },\n')
197
2012-10-21
fredg
198
2012-10-21
fredg
199
2012-10-21
fredg
# to XML DB
200
2012-10-21
fredg
def toxml(pkgDct):
201
2012-10-21
fredg
"""
202
2012-10-21
fredg
Export PACKAGES.TXT to a XML database format.
203
2012-10-21
fredg
"""
204
2012-10-29
fredg
with open(outputfile, 'a') as xmlf:
205
2012-10-21
fredg
xmlf.write('\t<package>\n')
206
2012-11-01
fredg
xmlf.write('\t\t<name>'
207
2012-11-01
fredg
+ pkgDct.get("name") + '</name>\n')
208
2012-11-01
fredg
xmlf.write('\t\t<version>'
209
2012-11-01
fredg
+ pkgDct.get("version") + '</version>\n')
210
2012-11-01
fredg
xmlf.write('\t\t<arch>'
211
2012-11-01
fredg
+ pkgDct.get("arch") + '</arch>\n')
212
2012-11-01
fredg
xmlf.write('\t\t<release>'
213
2012-11-01
fredg
+ pkgDct.get("release") + '</release>\n')
214
2012-11-01
fredg
xmlf.write('\t\t<location>'
215
2012-11-01
fredg
+ pkgDct.get("location") + '</location>\n')
216
2012-11-01
fredg
xmlf.write('\t\t<deps>'
217
2012-11-01
fredg
+ pkgDct.get("deps") + '</deps>\n')
218
2012-11-01
fredg
xmlf.write('\t\t<sizec>'
219
2012-11-01
fredg
+ pkgDct.get("sizec") + '</sizec>\n')
220
2012-11-01
fredg
xmlf.write('\t\t<sizeu>'
221
2012-11-01
fredg
+ pkgDct.get("sizeu") + '</sizeu>\n')
222
2012-11-01
fredg
xmlf.write('\t\t<slackdesc>'
223
2012-11-01
fredg
+ pkgDct.get("slackdesc") + '</slackdesc>\n')
224
2012-10-21
fredg
xmlf.write('\t</package>\n')
225
2012-10-21
fredg
226
2012-10-21
fredg
227
2012-10-21
fredg
# parser
228
2012-10-21
fredg
def mkdadb(towhat):
229
2012-10-21
fredg
"""
230
2012-10-21
fredg
Parse PACKAGES.TXT to get the values we need.
231
2012-10-21
fredg
Choose the export format:
232
2012-10-21
fredg
- CSV : tocsv
233
2012-10-21
fredg
- JSON : tojson
234
2012-10-21
fredg
- XML : toxml
235
2012-10-21
fredg
"""
236
2012-10-21
fredg
if towhat == tocsv:
237
2012-10-29
fredg
if os.path.isfile(outputfile):
238
2012-10-29
fredg
os.remove(outputfile)
239
2012-10-30
fredg
print outputfile, 'has been updated.'
240
2012-10-30
fredg
else:
241
2012-10-30
fredg
print outputfile, 'has been built.'
242
2012-10-21
fredg
if towhat == tojson:
243
2012-10-29
fredg
if os.path.isfile(outputfile):
244
2012-10-29
fredg
os.remove(outputfile)
245
2012-10-30
fredg
print outputfile, 'has been updated.'
246
2012-10-30
fredg
else:
247
2012-10-30
fredg
print outputfile, 'has been built.'
248
2012-10-22
fredg
with open("pre.json", 'w') as j:
249
2012-10-22
fredg
j.write('{\n')
250
2012-10-22
fredg
j.write('"packages": [\n')
251
2012-10-21
fredg
if towhat == toxml:
252
2012-10-29
fredg
if os.path.isfile(outputfile):
253
2012-10-29
fredg
os.remove(outputfile)
254
2012-10-30
fredg
print outputfile, 'has been updated.'
255
2012-10-30
fredg
else:
256
2012-10-30
fredg
print outputfile, 'has been built.'
257
2012-10-29
fredg
with open(outputfile, 'w') as xmlf:
258
2012-10-21
fredg
xmlf.write('<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n')
259
2012-10-21
fredg
xmlf.write('<packages>\n')
260
2012-10-21
fredg
pkg = new_pkgdct()
261
2012-10-21
fredg
with open('PACKAGES.TXT') as f:
262
2012-10-21
fredg
for line in f:
263
2012-10-21
fredg
pkgline = re.match(
264
2012-10-21
fredg
r'(PACKAGE NAME:\s\s)(.*)', line)
265
2012-10-21
fredg
locationline = re.match(
266
2012-10-21
fredg
r'(PACKAGE LOCATION:\s\s\.)(.*)', line)
267
2012-10-21
fredg
depline = re.match(
268
2012-10-21
fredg
r'(PACKAGE REQUIRED:\s\s)(.*)', line)
269
2012-10-21
fredg
sizecline = re.match(
270
2012-10-21
fredg
r'(PACKAGE\sSIZE\s\(compressed\):\s\s)(.*)', line)
271
2012-10-21
fredg
sizeuline = re.match(
272
2012-10-21
fredg
r'(PACKAGE\sSIZE\s\(uncompressed\):\s\s)(.*)', line)
273
2012-10-21
fredg
slackdescline = re.match(
274
2012-10-21
fredg
r'(%s:\s)(.*)' % pkg["name"].replace('+', '\+'), line)
275
2012-10-21
fredg
emptyline = re.match(
276
2012-10-21
fredg
r'^$', line)
277
2012-10-21
fredg
if pkgline:
278
2012-10-21
fredg
pname = pkgline.group(2)
279
2012-10-21
fredg
pname = re.match(
280
2012-10-21
fredg
r'(.*)-([^-]*)-([^-]*)-([^-]*).t[glx]z$', pname)
281
2012-10-21
fredg
pkg["name"] = pname.group(1)
282
2012-10-21
fredg
pkg["version"] = pname.group(2)
283
2012-10-21
fredg
pkg["arch"] = pname.group(3)
284
2012-10-21
fredg
pkg["release"] = pname.group(4)
285
2012-10-21
fredg
if depline:
286
2012-10-21
fredg
pkg["deps"] = depline.group(2)
287
2012-10-21
fredg
if locationline:
288
2012-10-21
fredg
pkg["location"] = locationline.group(2)
289
2012-10-21
fredg
if sizecline:
290
2012-10-21
fredg
pkg["sizec"] = sizecline.group(2)
291
2012-10-21
fredg
if sizeuline:
292
2012-10-21
fredg
pkg["sizeu"] = sizeuline.group(2)
293
2012-10-21
fredg
if slackdescline:
294
2012-10-21
fredg
pkg["slackdesc"] += " " + slackdescline.group(2).\
295
2012-10-21
fredg
replace('"', '\'').\
296
2012-10-21
fredg
replace('&', 'and').\
297
2012-10-21
fredg
replace('>', '').\
298
2012-10-21
fredg
replace('<', '')
299
2012-10-21
fredg
if emptyline and pkg.get("name"):
300
2012-10-21
fredg
pkg["slackdesc"] = pkg["slackdesc"].strip()
301
2012-10-21
fredg
towhat(pkg)
302
2012-10-21
fredg
pkg = new_pkgdct()
303
2012-10-21
fredg
if towhat == tojson:
304
2012-10-29
fredg
with open("pre.json", 'r') as j, open(outputfile, "w") as jsonf:
305
2012-10-22
fredg
alllines = j.readlines()
306
2012-10-22
fredg
alllines[-1] = alllines[-1].replace('},', '}')
307
2012-10-22
fredg
jsonf.writelines(alllines)
308
2012-10-22
fredg
jsonf.write(']\n')
309
2012-10-22
fredg
jsonf.write('}\n')
310
2012-10-22
fredg
os.remove("pre.json")
311
2012-10-21
fredg
if towhat == toxml:
312
2012-10-29
fredg
with open(outputfile, 'a') as xmlf:
313
2012-10-22
fredg
xmlf.write('</packages>\n')
314
2012-10-21
fredg
315
2012-10-21
fredg
316
2012-10-21
fredg
def main():
317
2012-10-29
fredg
if not update and not os.path.isfile(pkgtxt):
318
2012-11-01
fredg
sys.exit('No PACKAGES.TXT found, you should fetch one, aborting.')
319
2012-11-01
fredg
elif not update and os.path.isfile(pkgtxt):
320
2012-11-01
fredg
if repo:
321
2012-11-01
fredg
sys.exit("The repo variable can't be setup without --update, aborting.")
322
2012-11-01
fredg
if target:
323
2012-11-01
fredg
sys.exit("The target variable can't be setup without --update, aborting.")
324
2012-11-01
fredg
if release:
325
2012-11-01
fredg
sys.exit("The release variable can't be setup without --update, aborting.")
326
2012-10-29
fredg
else:
327
2012-10-29
fredg
pkgtxturl(repo, target, release, expa)
328
2012-10-29
fredg
329
2012-10-29
fredg
if convert == 'csv':
330
2012-10-29
fredg
mkdadb(tocsv)
331
2012-10-29
fredg
elif convert == 'json':
332
2012-10-29
fredg
mkdadb(tojson)
333
2012-10-29
fredg
elif convert == 'xml':
334
2012-10-29
fredg
mkdadb(toxml)
335
2012-10-29
fredg
else:
336
2012-10-29
fredg
sys.exit('You have to choose a valid database format, aborting.')
337
2012-10-29
fredg
338
2012-10-21
fredg
339
2012-10-21
fredg
if __name__ == '__main__':
340
2012-10-21
fredg
main()
Frédéric Galusik