1 """
2 CSB build related tools and programs.
3
4 When executed as a program, this module will run the CSB Build Console and
5 build the source tree it belongs to. The source tree is added at the
6 B{beginning} of sys.path to make sure that all subsequent imports from the
7 Test and Doc consoles will import the right thing (think of multiple CSB
8 packages installed on the same server).
9
10 Here is how to build, test and package the whole project::
11
12 $ hg clone https://hg.codeplex.com/csb CSB
13 $ CSB/csb/build.py -o <output directory>
14
15 The Console can also be imported and instantiated as a regular Python class.
16 In this case the Console again builds the source tree it is part of, but
17 sys.path will remain intact. Therefore, the Console will assume that all
18 modules currently in memory, as well as those that can be subsequently imported
19 by the Console itself, belong to the same CSB package.
20
21 @note: The CSB build services no longer support the option to build external
22 source trees.
23 @see: [CSB 0000038]
24 """
25 from __future__ import print_function
26
27 import os
28 import sys
29 import getopt
30 import traceback
31 import compileall
32
33 if os.path.basename(__file__) == '__init__.py':
34 PARENT = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
35 else:
36 PARENT = os.path.abspath(os.path.dirname(__file__))
37
38 ROOT = 'csb'
39 SOURCETREE = os.path.abspath(os.path.join(PARENT, ".."))
40
41 if __name__ == '__main__':
42
43
44
45 for path in sys.path:
46 if path.startswith(SOURCETREE):
47 sys.path.remove(path)
48
49 import io
50 assert hasattr(io, 'BufferedIOBase')
51
52 sys.path = [SOURCETREE] + sys.path
53
54
55 """
56 It is now safe to import any modules
57 """
58 import imp
59 import shutil
60 import tarfile
61
62 import csb
63
64 from abc import ABCMeta, abstractmethod
65 from csb.io import Shell
69 """
70 Enumeration of build types.
71 """
72
73 SOURCE = 'source'
74 BINARY = 'binary'
75
76 _du = { SOURCE: 'sdist', BINARY: 'bdist' }
77
78 @staticmethod
80 try:
81 return BuildTypes._du[key]
82 except KeyError:
83 raise ValueError('Unhandled build type: {0}'.format(key))
84
87 """
88 CSB Build Bot. Run with -h for usage.
89
90 @param output: build output directory
91 @type output: str
92 @param verbosity: verbosity level
93 @type verbosity: int
94
95 @note: The build console automatically detects and builds the csb package
96 it belongs to. You cannot build a different source tree with it.
97 See the module documentation for more info.
98 """
99
100 PROGRAM = __file__
101
102 USAGE = r"""
103 CSB Build Console: build, test and package the entire csb project.
104
105 Usage:
106 python {program} -o output [-v verbosity] [-t type] [-h]
107
108 Options:
109 -o output Build output directory
110 -v verbosity Verbosity level, default is 1
111 -t type Build type:
112 source - build source code distribution (default)
113 binary - build executable
114 -h, --help Display this help
115 """
116
118
119 self._input = None
120 self._output = None
121 self._temp = None
122 self._docs = None
123 self._apidocs = None
124 self._root = None
125 self._verbosity = None
126 self._type = buildtype
127 self._dist = BuildTypes.get(buildtype)
128
129 if os.path.join(SOURCETREE, ROOT) != PARENT:
130 raise IOError('{0} must be a sub-package or sub-module of {1}'.format(__file__, ROOT))
131 self._input = SOURCETREE
132
133 self._success = True
134
135 self.output = output
136 self.verbosity = verbosity
137
138 @property
141
142 @property
145 @output.setter
147
148 self._output = os.path.abspath(value)
149 self._temp = os.path.join(self._output, 'build')
150 self._docs = os.path.join(self._temp, 'docs')
151 self._apidocs = os.path.join(self._docs, 'api')
152 self._root = os.path.join(self._temp, ROOT)
153
154 @property
156 return self._verbosity
157 @verbosity.setter
159 self._verbosity = int(value)
160
162 """
163 Run the console.
164 """
165 self.log('\n# Building package {0} from {1}\n'.format(ROOT, SOURCETREE))
166
167 self._init()
168 v = self._revision()
169 self._doc(v)
170 self._test()
171
172 self._compile()
173 vn = self._package()
174
175 if self._success:
176 self.log('\n# Done ({0}).\n'.format(vn.full))
177 else:
178 self.log('\n# Build failed.\n')
179
180
181 - def log(self, message, level=1, ending='\n'):
182
183 if self._verbosity >= level:
184 sys.stdout.write(message)
185 sys.stdout.write(ending)
186 sys.stdout.flush()
187
189 """
190 Collect all required stuff in the output folder.
191 """
192 self.log('# Preparing the file system...')
193
194 if not os.path.exists(self._output):
195 self.log('Creating output directory {0}'.format(self._output), level=2)
196 os.mkdir(self._output)
197
198 if os.path.exists(self._temp):
199 self.log('Deleting existing temp directory {0}'.format(self._temp), level=2)
200 shutil.rmtree(self._temp)
201
202 self.log('Copying the source tree to temp directory {0}'.format(self._temp), level=2)
203 shutil.copytree(self._input, self._temp)
204
205 if os.path.exists(self._apidocs):
206 self.log('Deleting existing API docs directory {0}'.format(self._apidocs), level=2)
207 shutil.rmtree(self._apidocs)
208 if not os.path.isdir(self._docs):
209 self.log('Creating docs directory {0}'.format(self._docs), level=2)
210 os.mkdir(self._docs)
211 self.log('Creating API docs directory {0}'.format(self._apidocs), level=2)
212 os.mkdir(self._apidocs)
213
232
260
261 - def _doc(self, version):
262 """
263 Build documentation in the output folder.
264 """
265 self.log('\n# Generating API documentation...')
266 try:
267 import epydoc.cli
268 except ImportError:
269 self.log('\n Skipped: epydoc is missing')
270 return
271
272 self.log('\n# Emulating ARGV for the Doc Builder...', level=2)
273 argv = sys.argv
274 sys.argv = ['epydoc', '--html', '-o', self._apidocs,
275 '--name', '{0} v{1}'.format(ROOT.upper(), version),
276 '--no-private', '--introspect-only', '--exclude', 'csb.test.cases',
277 '--css', os.path.join(self._temp, 'epydoc.css'),
278 '--fail-on-error', '--fail-on-warning', '--fail-on-docstring-warning',
279 self._root]
280
281 if self._verbosity > 0:
282 sys.argv.append('-v')
283
284 try:
285 epydoc.cli.cli()
286 sys.exit(0)
287 except SystemExit as ex:
288 if ex.code is 0:
289 self.log('\n Passed all doc tests')
290 else:
291 if ex.code == 2:
292 self.log('\n DID NOT PASS: The docs might be broken')
293 else:
294 self.log('\n FAIL: Epydoc returned "#{0.code}: {0}"'.format(ex))
295 self._success = False
296
297 self.log('\n# Restoring the previous ARGV...', level=2)
298 sys.argv = argv
299
301 """
302 Byte-compile all modules and packages.
303 """
304 self.log('\n# Byte-compiling all *.py files...')
305
306 quiet = self.verbosity <= 1
307 valid = compileall.compile_dir(self._root, quiet=quiet, force=True)
308
309 if not valid:
310 self.log('\n FAIL: Compilation error(s)\n')
311 self._success = False
312
314 """
315 Make package.
316 """
317 self.log('\n# Configuring CWD and ARGV for the Setup...', level=2)
318 cwd = os.curdir
319 os.chdir(self._temp)
320
321 if self._verbosity > 1:
322 verbosity = '-v'
323 else:
324 verbosity = '-q'
325 argv = sys.argv
326 sys.argv = ['setup.py', verbosity, self._dist, '-d', self._output]
327
328 self.log('\n# Building {0} distribution...'.format(self._type))
329 try:
330 setup = imp.load_source('setupcsb', 'setup.py')
331 d = setup.build()
332 version = setup.VERSION
333 package = d.dist_files[0][2]
334
335 if self._type == BuildTypes.BINARY:
336 self._strip_source(package)
337
338 except SystemExit as ex:
339 if ex.code is not 0:
340 self.log('\n FAIL: Setup returned: \n\n{0}\n'.format(ex))
341 self._success = False
342 package = 'FAIL'
343
344 self.log('\n# Restoring the previous CWD and ARGV...', level=2)
345 os.chdir(cwd)
346 sys.argv = argv
347
348 self.log(' Packaged ' + package)
349 return version
350
352 """
353 Delete plain text source code files from the package.
354 """
355 cwd = os.getcwd()
356
357 try:
358 tmp = os.path.join(self.output, 'tmp')
359 os.mkdir(tmp)
360
361 self.log('\n# Entering {1} in order to delete .py files from {0}...'.format(package, tmp), level=2)
362 os.chdir(tmp)
363
364 oldtar = tarfile.open(package, mode='r:gz')
365 oldtar.extractall(tmp)
366 oldtar.close()
367
368 newtar = tarfile.open(package, mode='w:gz')
369
370 try:
371 for i in os.walk('.'):
372 for fn in i[2]:
373 if fn.endswith('.py'):
374 module = os.path.join(i[0], fn);
375 if not os.path.isfile(module.replace('.py', '.pyc')):
376 raise ValueError('Missing bytecode for module {0}'.format(module))
377 else:
378 os.remove(os.path.join(i[0], fn))
379
380 for i in os.listdir('.'):
381 newtar.add(i)
382 finally:
383 newtar.close()
384
385 finally:
386 self.log('\n# Restoring the previous CWD...', level=2)
387 os.chdir(cwd)
388 if os.path.exists(tmp):
389 shutil.rmtree(tmp)
390
391 @staticmethod
392 - def exit(message=None, code=0, usage=True):
400
401 @staticmethod
402 - def run(argv=None):
403
404 if argv is None:
405 argv = sys.argv[1:]
406
407 output = None
408 verb = 1
409 buildtype = BuildTypes.SOURCE
410
411 try:
412 options, dummy = getopt.getopt(argv, 'o:v:t:h', ['output=', 'verbosity=', 'type=', 'help'])
413
414 for option, value in options:
415 if option in('-h', '--help'):
416 Console.exit(message=None, code=0)
417 if option in('-o', '--output'):
418 if not os.path.isdir(value):
419 Console.exit(message='E: Output directory not found "{0}".'.format(value), code=3)
420 output = value
421 if option in('-v', '--verbosity'):
422 try:
423 verb = int(value)
424 except ValueError:
425 Console.exit(message='E: Verbosity must be an integer.', code=4)
426 if option in('-t', '--type'):
427 if value not in [BuildTypes.SOURCE, BuildTypes.BINARY]:
428 Console.exit(message='E: Invalid build type "{0}".'.format(value), code=5)
429 buildtype = value
430 except getopt.GetoptError as oe:
431 Console.exit(message='E: ' + str(oe), code=1)
432
433 if not output:
434 Console.exit(code=1, usage=True)
435 else:
436 try:
437 Console(output, verbosity=verb, buildtype=buildtype).build()
438 except Exception as ex:
439 msg = 'Unexpected Error: {0}\n\n{1}'.format(ex, traceback.format_exc())
440 Console.exit(message=msg, code=99, usage=False)
441
450
452 """
453 Determines the current repository revision number of a working copy.
454
455 @param path: a local checkout path to be examined
456 @type path: str
457 @param sc: name of the source control program
458 @type sc: str
459 """
460
462
463 self._path = None
464 self._sc = None
465
466 if os.path.exists(path):
467 self._path = path
468 else:
469 raise IOError('Path not found: {0}'.format(path))
470 if Shell.run([sc, 'help']).code is 0:
471 self._sc = sc
472 else:
473 raise RevisionError('Source control binary probe failed', None, None)
474
475 @property
478
479 @property
482
483 @abstractmethod
485 """
486 Return the current revision information.
487 @rtype: L{RevisionInfo}
488 """
489 pass
490
491 - def write(self, revision, sourcefile):
492 """
493 Finalize the __version__ = major.minor.micro.{revision} tag.
494 Overwrite C{sourcefile} in place by substituting the {revision} macro.
495
496 @param revision: revision number to write to the source file.
497 @type revision: int
498 @param sourcefile: python source file with a __version__ tag, typically
499 "csb/__init__.py"
500 @type sourcefile: str
501
502 @return: sourcefile.__version__
503 """
504 content = open(sourcefile).readlines()
505
506 with open(sourcefile, 'w') as src:
507 for line in content:
508 if line.startswith('__version__'):
509 src.write(line.format(revision=revision))
510 else:
511 src.write(line)
512
513 self._delcache(sourcefile)
514 return imp.load_source('____source', sourcefile).__version__
515
516 - def _run(self, cmd):
517
518 si = Shell.run(cmd)
519 if si.code > 0:
520 raise RevisionError('SC failed ({0.code}): {0.stderr}'.format(si), si.code, si.cmd)
521
522 return si.stdout.splitlines()
523
525
526 compiled = os.path.splitext(sourcefile)[0] + '.pyc'
527 if os.path.isfile(compiled):
528 os.remove(compiled)
529
530 pycache = os.path.join(os.path.dirname(compiled), '__pycache__')
531 if os.path.isdir(pycache):
532 shutil.rmtree(pycache)
533
538
540
541 cmd = '{0.sc} info {0.path} -R'.format(self)
542 maxrevision = None
543
544 for line in self._run(cmd):
545 if line.startswith('Revision:'):
546 rev = int(line[9:] .strip())
547 if rev > maxrevision:
548 maxrevision = rev
549
550 if maxrevision is None:
551 raise RevisionError('No revision number found', code=0, cmd=cmd)
552
553 return RevisionInfo(self.path, maxrevision)
554
556
563
565
566 wd = os.getcwd()
567 os.chdir(self.path)
568
569 try:
570 cmd = '{0.sc} log -r tip'.format(self)
571
572 revision = None
573 changeset = ''
574
575 for line in self._run(cmd):
576 if line.startswith('changeset:'):
577 items = line[10:].split(':')
578 revision = int(items[0])
579 changeset = items[1].strip()
580 break
581
582 if revision is None:
583 raise RevisionError('No revision number found', code=0, cmd=cmd)
584
585 return RevisionInfo(self.path, revision, changeset)
586
587 finally:
588 os.chdir(wd)
589
591
592 - def __init__(self, item, revision, id=None):
597
601
602
603 if __name__ == '__main__':
604
605 main()
606