Commit Diff
Diff:
bc8d5284c3e4ba06c0900c440a984bb1d53f9c63
06f50514462c7bb80c7b4b9c884bb8b7986d53fd
Commit:
06f50514462c7bb80c7b4b9c884bb8b7986d53fd (tags/0.2.2)
Tree:
c1344f1649cfe04b85e0f7b6cd6e229576d433ad
Author:
frédéric galusik <fredg@salixos.org>
Committer:
frédéric galusik <fredg@salixos.org>
Date:
Thu May 2 13:28:13 2013 UTC
Message:
removed test file
blob - 2afe4842e928a2b636fbecddaeb8a69d1de1a2c5 (mode 644)
blob + /dev/null
--- pkgtxt2db2.py
+++ /dev/null
@@ -1,324 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-#
-# pkgtxt2db.py
-#
-# Copyright 2012 Frédéric Galusik <fredg~at~salixos~dot~org>
-#
-#
-# License: BSD Revised
-#
-# Convert the Slackware/Salix PACKAGES.TXT file to some various database
-# formats : CSV, json, xml
-#
-
-
-import os
-import sys
-import urllib2
-import gzip
-import re
-import argparse
-
-
-# Program information
-my_url = 'http://www.salixos.org/wiki/index.php/Pkgtxt2db'
-my_name = 'pkgtxt2db'
-my_version = '0.0'
-
-
-def main():
-# Parse the CLI options
- parser = argparse.ArgumentParser(
- prog='pkgtxt2db',
- description='Convert PACKAGES.TXT to DB',
- epilog=
- "i.e. pkgtxt2db -u -t salix -a x86_64 -r 14.0 -c json -o salix64.json")
-
- parser.add_argument('-u', '--update', action="store_true",
- default=False,
- help='Download/update the PACKAGES.TXT file')
-
- parser.add_argument('-t', '--target', action="store",
- dest='target', default='salix',
- help='Choose the O.S.: slackware or salix (default) ')
-
- parser.add_argument('--repo', action="store",
- dest='repo', default='i486',
- help='Choose the arch repo: x86_64 or i486 (default)')
-
- parser.add_argument('-e', '--expa', action="store",
- dest='expa', default='/',
- help='Choose the slackware extra/patches')
-
- parser.add_argument('-r', '--release', action="store",
- dest='release', default='14.0',
- help='Choose the release: 13.0 to 14.0 (default)')
-
- parser.add_argument('-c', '--convert', action="store",
- dest='convert',
- help='Choose the DB format: xml, json, csv')
-
- parser.add_argument('-o', '--output', action="store",
- dest='output', default='packages',
- help='Choose the name of your DB file')
-
- parser.add_argument('--version', action='version',
- version='%(prog)s ' + my_version)
-
- if len(sys.argv) == 1:
- sys.exit('Wrong usage, see pkgtxt2db --help')
-
- args = parser.parse_args()
-
- if not update and not os.path.isfile(pkgtxt):
- sys.exit('No PACKAGES.TXT found, you should fetch one, aborting.')
- else:
- pkgtxturl(repo, target, release, expa)
-
- if convert == 'csv':
- mkdadb(tocsv)
- elif convert == 'json':
- mkdadb(tojson)
- elif convert == 'xml':
- mkdadb(toxml)
- else:
- sys.exit('You have to choose a valid database format, aborting.')
-
-
-
- # vars
- mirror = 'http://salix.enialis.net/'
- pkgtxtz = 'PACKAGES.TXT.gz'
- pkgtxt = 'PACKAGES.TXT'
- update = args.update
- target = args.target
- repo = args.repo
- release = args.release
- expa = args.expa
- convert = args.convert
- output = args.output
- outputfile = '.'.join([output, convert])
-
-
-# initialise the pkg dictionnary with empty values
-fields = ['name',
- 'version',
- 'arch',
- 'release'
- 'location',
- 'deps',
- 'sizec',
- 'sizeu',
- 'slackdesc']
-
-
-def new_pkgdct():
- return dict(zip(fields, ['', ] * len(fields)))
-
-
-# Fetch PACKAGES.TXT
-def pkgtxturl(repo='i486', target='salix', release='14.0', expa='/'):
- """
- Download the slackware/salix PACKAGES.TXT.gz from a built URL and unzip it.
- pkgtxturl(repo, target, release, |extra|patches)
- """
-
- # Check the repo
- Rep = {'i486', 'x86_64'}
- if repo not in Rep:
- sys.exit('Choose a valid arch please, aborting.')
-
- # Check if the choosen release is a valid one.
- R = {'13.0', '13.1', '13.37', '14.0', 'current'}
- if release not in R:
- sys.exit('Choose a valid release please, aborting.')
-
- # Check the target
- if target == 'slackware':
- target = 'slackware-'
- elif target == 'salix':
- target = ''
- else:
- sys.exit('Choose a valid target, aborting.')
-
- #
- if expa == 'extra':
- expa = '/extra/'
- elif expa == 'patches':
- expa = '/patches/'
-
- slash = '/'
-
- # Build the URL to fetch PACKAGES.TXT
- url = mirror + repo + slash + target + release + expa + pkgtxtz
-
- # remove old files
- if os.path.isfile(pkgtxtz):
- os.remove(pkgtxtz)
- print "Remove old ", pkgtxtz
- if os.path.isfile(pkgtxt):
- os.remove(pkgtxt)
- print "Remove old ", pkgtxt
- try:
- f = urllib2.urlopen(url)
- print "Fetching ", url
- print ""
- # Open local_file for writing
- with open(os.path.basename(url), "wb") as local_file:
- local_file.write(f.read())
- except urllib2.HTTPError, e:
- print "HTTP Error:", e.code, url
- return False
- except urllib2.URLError, e:
- print "URL Error:", e.reason, url
- return False
-
- # unzip it
- fout = open(pkgtxt, 'w')
- with gzip.open(pkgtxtz, 'rb') as f:
- for line in f:
- fout.write(line)
- fout.close()
-
-
-# to CSV DB
-def tocsv(pkgDct, sep=";"):
- """
- Export PACKAGES.TXT to a CSV database format.
- The separated string can be choosen with the sep var, default is ;
- """
- with open(outputfile, 'a') as csvf:
- csvf.write(
- sep.join(map(lambda field: pkgDct.get(field, ''), fields)) + '\n')
-
-
-# to JSON DB
-def tojson(pkgDct):
- """
- Export PACKAGES.TXT to a JSON database format
- """
- with open("pre.json", 'a') as j:
- j.write(' {\n')
- j.write(' \"name\": \"' + pkgDct.get("name") + '\",\n')
- j.write(' \"version\": \"' + pkgDct.get("version") + '\",\n')
- j.write(' \"arch\": \"' + pkgDct.get("arch") + '\",\n')
- j.write(' \"release\": \"' + pkgDct.get("release") + '\",\n')
- j.write(' \"location\": \"' + pkgDct.get("location") + '\",\n')
- j.write(' \"deps\": \"' + pkgDct.get("deps") + '\",\n')
- j.write(' \"sizec\": \"' + pkgDct.get("sizec") + '\",\n')
- j.write(' \"sizeu\": \"' + pkgDct.get("sizeu") + '\",\n')
- j.write(' \"slackdesc\": \"' + pkgDct.get("slackdesc") + '\"\n')
- j.write(' },\n')
-
-
-# to XML DB
-def toxml(pkgDct):
- """
- Export PACKAGES.TXT to a XML database format.
- """
- with open(outputfile, 'a') as xmlf:
- xmlf.write('\t<package>\n')
- xmlf.write('\t\t<name>' + pkgDct.get("name") + '</name>\n')
- xmlf.write('\t\t<version>' + pkgDct.get("version") + '</version>\n')
- xmlf.write('\t\t<arch>' + pkgDct.get("arch") + '</arch>\n')
- xmlf.write('\t\t<release>' + pkgDct.get("release") + '</release>\n')
- xmlf.write('\t\t<location>' + pkgDct.get("location") + '</location>\n')
- xmlf.write('\t\t<deps>' + pkgDct.get("deps") + '</deps>\n')
- xmlf.write('\t\t<sizec>' + pkgDct.get("sizec") + '</sizec>\n')
- xmlf.write('\t\t<sizeu>' + pkgDct.get("sizeu") + '</sizeu>\n')
- xmlf.write('\t\t<slackdesc>' + pkgDct.get("slackdesc") + '</slackdesc>\n')
- xmlf.write('\t</package>\n')
-
-
-# parser
-def mkdadb(towhat):
- """
- Parse PACKAGES.TXT to get the values we need.
- Choose the export format:
- - CSV : tocsv
- - JSON : tojson
- - XML : toxml
- """
- if towhat == tocsv:
- if os.path.isfile(outputfile):
- os.remove(outputfile)
- print outputfile, 'has been updated.'
- else:
- print outputfile, 'has been built.'
- if towhat == tojson:
- if os.path.isfile(outputfile):
- os.remove(outputfile)
- print outputfile, 'has been updated.'
- else:
- print outputfile, 'has been built.'
- with open("pre.json", 'w') as j:
- j.write('{\n')
- j.write('"packages": [\n')
- if towhat == toxml:
- if os.path.isfile(outputfile):
- os.remove(outputfile)
- print outputfile, 'has been updated.'
- else:
- print outputfile, 'has been built.'
- with open(outputfile, 'w') as xmlf:
- xmlf.write('<?xml version=\"1.0\" encoding=\"UTF-8\" ?>\n')
- xmlf.write('<packages>\n')
- pkg = new_pkgdct()
- with open('PACKAGES.TXT') as f:
- for line in f:
- pkgline = re.match(
- r'(PACKAGE NAME:\s\s)(.*)', line)
- locationline = re.match(
- r'(PACKAGE LOCATION:\s\s\.)(.*)', line)
- depline = re.match(
- r'(PACKAGE REQUIRED:\s\s)(.*)', line)
- sizecline = re.match(
- r'(PACKAGE\sSIZE\s\(compressed\):\s\s)(.*)', line)
- sizeuline = re.match(
- r'(PACKAGE\sSIZE\s\(uncompressed\):\s\s)(.*)', line)
- slackdescline = re.match(
- r'(%s:\s)(.*)' % pkg["name"].replace('+', '\+'), line)
- emptyline = re.match(
- r'^$', line)
- if pkgline:
- pname = pkgline.group(2)
- pname = re.match(
- r'(.*)-([^-]*)-([^-]*)-([^-]*).t[glx]z$', pname)
- pkg["name"] = pname.group(1)
- pkg["version"] = pname.group(2)
- pkg["arch"] = pname.group(3)
- pkg["release"] = pname.group(4)
- if depline:
- pkg["deps"] = depline.group(2)
- if locationline:
- pkg["location"] = locationline.group(2)
- if sizecline:
- pkg["sizec"] = sizecline.group(2)
- if sizeuline:
- pkg["sizeu"] = sizeuline.group(2)
- if slackdescline:
- pkg["slackdesc"] += " " + slackdescline.group(2).\
- replace('"', '\'').\
- replace('&', 'and').\
- replace('>', '').\
- replace('<', '')
- if emptyline and pkg.get("name"):
- pkg["slackdesc"] = pkg["slackdesc"].strip()
- towhat(pkg)
- pkg = new_pkgdct()
- if towhat == tojson:
- with open("pre.json", 'r') as j, open(outputfile, "w") as jsonf:
- alllines = j.readlines()
- alllines[-1] = alllines[-1].replace('},', '}')
- jsonf.writelines(alllines)
- jsonf.write(']\n')
- jsonf.write('}\n')
- os.remove("pre.json")
- if towhat == toxml:
- with open(outputfile, 'a') as xmlf:
- xmlf.write('</packages>\n')
-
-
-if __name__ == '__main__':
- main()
Frédéric Galusik