diff --git a/README.md b/README.md index 1ca3989..b211213 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,13 @@ ktt === -Kai's text tools \ No newline at end of file +Kai's text tools + +Contains following python modules: +atg - Advanced Text Generator. Created to generate large numbers of files, using a template and a data stored in table. +atr - Advanced Text Replacer. Created to handle various ways of replacements in text files. +template - template class for ATG and ATR. +data - data class for ATG and ATR. + +Scripts: +ktt_atgcsv.py - generates files from CSV table and ATGv2 template file. \ No newline at end of file diff --git a/atg.py b/atg.py new file mode 100644 index 0000000..624f3b0 --- /dev/null +++ b/atg.py @@ -0,0 +1,62 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +''' +Advanced Text Generator module for a Kai's Text Tools. + +(c) 2013 Ivan "Kai SD" Korystin + +License: GPLv3 +''' +from os.path import join, split, exists +from os import makedirs + +class ATG(object): + ''' + Advanced Text Generator is a class, created to generate multiple text files from table data. + ''' + def __init__(self, data, template): + self.data = data + self.template = template + self.out = template.process(data) + + if type(self.out) == dict: + self.multiple = True + else: + self.multiple = False + + def write_files(self, outputDir='.'): + encoding = self.template.encoding + extension = self.template.extension + out = self.out + if self.multiple: + for name in out.keys(): + namepath = name.replace('\\', '/').split('/') + newpath = u'' + for i in namepath[:-1]: + newpath = join(newpath, i) + if not exists(join(unicode(outputDir),newpath)): + makedirs(join(unicode(outputDir),newpath)) + fname = join(unicode(outputDir),name+'.'+extension) + if fname.endswith('.'): + fname = fname[:-1] + f = open(fname, 'w') + f.write(out[name].encode(encoding)) + self.log(' Saved %s' % (name+'.'+extension)) + f.close() + else: + name = self.template.bonusPrefix + '.' + extension + if name == '.': + name = self.template.keyField + namepath = name.replace('\\', '/').split('/') + newpath = u'' + for i in namepath[:-1]: + newpath = join(newpath, i) + if not exists(join(unicode(outputDir),newpath)): + makedirs(join(unicode(outputDir),newpath)) + f = open(join(unicode(outputDir),name+'.'+extension), 'w') + f.write(out.encode(encoding)) + self.log(' Saved %s' % (name+'.'+extension)) + f.close() + + def log(self, text): + pass \ No newline at end of file diff --git a/atr.py b/atr.py new file mode 100644 index 0000000..36f1ae4 --- /dev/null +++ b/atr.py @@ -0,0 +1,163 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +''' +Advanced Text Replacer module for a Kai's Text Tools. + +(c) 2013 Ivan "Kai SD" Korystin + +License: GPLv3 +''' +import re +class ATR(object): + ''' + classdocs + ''' + + def __init__(self, files): + ''' + Constructor + ''' + self.files = files + self.replacements = [] + + def plain_replace(self, pattern, string, regexp=False): + ''' + Replaces the given pattern with string in files. + ''' + if regexp: + pattern = re.compile(pattern) + self.replacements.append((pattern, string)) + + + def templated_replace(self, pattern, template, data, keyFormat='filename', regexp=False): + ''' + Replaces the given pattern with data formated by template. + Valid values for keyFormat: + filename - take data rows by filename(path ignored), key value of the data row should store the filename. + fullname - as filename, but with path. + index - take data rows in order, key value of the data row should store the index. Indexes starts with 0. + If filename or index cannot be found in data keys, pattern will not be replaced. + ''' + if regexp: + pattern = re.compile(pattern) + strings = template.process(data) + self.replacements.append((pattern, strings, keyFormat)) + + + def write_in_place(self): + ''' + Do replacement and save the files + ''' + for f in self.files: + out = u'' + with open(f, 'rb') as file: + out = file.read() + + idx = 0 + for r in self.replacements: + if type(r[0]) in (str, unicode): + pattern = re.compile(re.escape(r[0])) + string = r[1] + elif type(r[0]) is dict and len(r) == 3: + if r[2] == 'filename': + fname = f.replace('\\', '/').split('/')[-1] + string = f[1].get(fname, None) + elif r[2] == 'fullname': + string = f[1].get(f, None) + elif r[2] == 'index': + fname = f.replace('\\', '/').split('/')[-1] + string = f[1].get(idx, None) + else: + raise BaseException('Unknown data key format.') + elif hasattr(r[0], 'match'): + pattern = r[0] + string = r[1] + else: + raise BaseException('Unknown pattern type.') + if string: + out = re.sub(pattern, string, out) + + with open(f, 'wb') as outfile: + outfile.write(out) + + def write_new_files(self, outfiles): + ''' + Do replacement, but save to given files instead of the original ones. + ''' + if not len(outfiles) == len(self.files): + raise BaseException('Lists of original and new files has different length.') + + for f in self.files: + out = u'' + with open(f, 'rb') as file: + out = file.read() + + idx = 0 + for r in self.replacements: + if type(r[0]) in (str, unicode): + pattern = re.compile(re.escape(r[0])) + string = r[1] + elif type(r[0]) is dict and len(r) == 3: + if r[2] == 'filename': + fname = f.replace('\\', '/').split('/')[-1] + string = f[1].get(fname, None) + elif r[2] == 'fullname': + string = f[1].get(f, None) + elif r[2] == 'index': + fname = f.replace('\\', '/').split('/')[-1] + string = f[1].get(idx, None) + else: + raise BaseException('Unknown data key format.') + elif hasattr(r[0], 'match'): + pattern = r[0] + string = r[1] + else: + raise BaseException('Unknown pattern type.') + if string: + out = re.sub(pattern, string, out) + + with open(outfiles[self.files.index(f)], 'wb') as outfile: + outfile.write(out) + + def replace_in_names(self): + ''' + Do replacement, but in file names instead of file content. Returns the list of new file names, + you can use it with writeNewFiles() method. + ''' + out = [] + for f in self.files: + new = f + idx = 0 + for r in self.replacements: + if type(r[0]) in (str, unicode): + pattern = re.compile(re.escape(r[0])) + string = r[1] + elif type(r[0]) is dict and len(r) == 3: + if r[2] == 'filename': + fname = f.replace('\\', '/').split('/')[-1] + string = f[1].get(fname, None) + elif r[2] == 'fullname': + string = f[1].get(f, None) + elif r[2] == 'index': + fname = f.replace('\\', '/').split('/')[-1] + string = f[1].get(idx, None) + else: + raise BaseException('Unknown data key format.') + elif hasattr(r[0], 'match'): + pattern = r[0] + string = r[1] + else: + raise BaseException('Unknown pattern type.') + if string: + new = re.sub(pattern, string, new) + out.append(new) + return out + + def clear_replacements(self): + ''' + Removes all replacements. + ''' + self.replacements = [] + + def log(self, string): + pass \ No newline at end of file diff --git a/data.py b/data.py new file mode 100644 index 0000000..0163062 --- /dev/null +++ b/data.py @@ -0,0 +1,176 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +''' +Data module for a Kai's Text Tools. + +(c) 2013 Ivan "Kai SD" Korystin + +License: GPLv3 +''' + +import csv, codecs + +class Data(object): + ''' + Empty data class. Can be used for a subclassing or procedural data creation. + ''' + def __init__(self, *args, **kwargs): + ''' + Constructor + ''' + self.keys = [] + self.rows = [] + + def __getitem__(self, pair): + ''' + Returns a value for given key and row. + ''' + key = pair[0] + row = pair[1] + + keys = self.keys + rows = self.rows + if key in keys: + if len(rows) > row: + return rows[row][keys.index(key)] + else: + raise BaseException('Row %i not found in data' % (row)) + else: + raise BaseException('Named value %s not found in data' % (key)) + + def __setitem__(self, pair, value): + ''' + Sets a value for given key and row. + ''' + key = pair[0] + row = pair[1] + + keys = self.keys + rows = self.rows + if key in keys: + if len(rows) > row: + rows[row][keys.index(key)] = value + else: + raise BaseException('Row %i not found in data' % (row)) + else: + raise BaseException('Named value %s not found in data' % (key)) + + def __str__(self): + ''' + Returns data as string. + ''' + return str((self.keys, self.rows)) + + def __repr__(self): + return self.__str__() + + def has_key(self, key): + ''' + Returns True if given key exists in data + ''' + return key in self.keys + + def add_rows(self, n=1): + ''' + Adds some empty rows to the data. + ''' + keys = self.keys + rows = self.rows + + for n in xrange(0, n): + row = [] + for k in keys: + row.append('') + rows.append(row) + + def add_keys(self, *h): + ''' + Adds new keys to the data. + ''' + keys = self.keys + rows = self.rows + + for i in h: + keys.append(i) + for r in rows: + for i in h: + r.append('') + + def col_by_key(self, key): + cols = [] + keys = self.keys + rows = self.rows + if key in keys: + idx = keys.index(key) + for r in rows: + cols.append(r[idx]) + else: + raise BaseException('Named value %s not found in data' % (key)) + return tuple(cols) + + def row_by_idx(self, idx): + return tuple(self.rows[idx]) + +class CSVData(Data): + ''' + Class for reading CSV files. + ''' + class Reader: + class Recoder: + def __init__(self, f, encoding): + self.reader = codecs.getreader(encoding)(f) + + def __iter__(self): + return self + + def next(self): + return self.reader.next().encode("utf-8") + + def __init__(self, f, dialect=csv.excel, encoding="utf-8", **kwds): + f = self.Recoder(f, encoding) + self.reader = csv.reader(f, dialect=dialect, **kwds) + + def next(self): + row = self.reader.next() + return [unicode(s, "utf-8") for s in row] + + def __iter__(self): + return self + + def __init__(self, filename, encoding='utf-8', delimiter=';', quotechar='"', **kwargs): + csvfile = self.Reader(open(filename), encoding=encoding, delimiter=delimiter, quotechar=quotechar) + sourceData = [] + sourcekeys = None + + if kwargs.get('transpose', False): + sourcekeys = [] + rowData = [] + for i in csvfile: + sourcekeys.append(i[0]) + for k in xrange(1, len(i)): + sourceData.append([]) + try: + i[k] = int(i[k]) + except: + try: + i[k] = float(i[k]) + except: + i[k] = i[k] + rowData.append(i[1:]) + sourceData = list(map(lambda *x:x, *rowData)) + else: + for i in csvfile: + if sourcekeys is None: + sourcekeys = i + else: + for k in xrange(0, len(i)): + try: + i[k] = int(i[k]) + except: + try: + i[k] = float(i[k]) + except: + i[k] = i[k] + sourceData.append(i) + self.keys = sourcekeys + self.rows = sourceData \ No newline at end of file diff --git a/ktt_atgcsv.py b/ktt_atgcsv.py new file mode 100755 index 0000000..b026b84 --- /dev/null +++ b/ktt_atgcsv.py @@ -0,0 +1,27 @@ +#!/usr/bin/python +# -*- coding: utf-8 -*- +''' +Generates files from csv table. + +Part of Kai's Text Tools + +(c) 2013 Ivan "Kai SD" Korystin + +License: GPLv3 +''' +from sys import argv +from atg import ATG +from data import CSVData +from template import TemplateV2 +from os.path import split + +if __name__ == '__main__': + if len(argv) == 3: + generator = ATG(CSVData(argv[1]), TemplateV2(argv[2])) + generator.write_files() + elif len(argv) == 4: + generator = ATG(CSVData(argv[1]), TemplateV2(argv[2])) + generator.write_files(argv[3]) + else: + print 'Usage:', split(argv[0])[-1], '', '