| 8 | | import scrapy |
| 9 | | from scrapy import log |
| 10 | | from scrapy.xlib import lsprofcalltree |
| 11 | | from scrapy.conf import settings |
| 12 | | from scrapy.command.models import ScrapyCommand |
| 13 | | from scrapy.utils.signal import send_catch_log |
| 14 | | |
| 15 | | # Signal that carries information about the command which was executed |
| 16 | | # args: cmdname, cmdobj, args, opts |
| 17 | | command_executed = object() |
| 18 | | |
| 19 | | def _find_commands(dir): |
| 20 | | try: |
| 21 | | return [f[:-3] for f in os.listdir(dir) if not f.startswith('_') and \ |
| 22 | | f.endswith('.py')] |
| 23 | | except OSError: |
| 24 | | return [] |
| 25 | | |
| 26 | | def _get_commands_from_module(module): |
| 27 | | d = {} |
| 28 | | mod = __import__(module, {}, {}, ['']) |
| 29 | | for cmdname in _find_commands(mod.__path__[0]): |
| 30 | | modname = '%s.%s' % (module, cmdname) |
| 31 | | command = getattr(__import__(modname, {}, {}, [cmdname]), 'Command', None) |
| 32 | | if callable(command): |
| 33 | | d[cmdname] = command() |
| 34 | | else: |
| 35 | | print 'WARNING: Module %r does not define a Command class' % modname |
| 36 | | return d |
| 37 | | |
| 38 | | def _get_commands_dict(): |
| 39 | | cmds = _get_commands_from_module('scrapy.commands') |
| 40 | | cmds_module = settings['COMMANDS_MODULE'] |
| 41 | | if cmds_module: |
| 42 | | cmds.update(_get_commands_from_module(cmds_module)) |
| 43 | | return cmds |
| 44 | | |
| 45 | | def _get_command_name(argv): |
| 46 | | for arg in argv[1:]: |
| 47 | | if not arg.startswith('-'): |
| 48 | | return arg |
| 49 | | |
| 50 | | def _print_usage(inside_project): |
| 51 | | if inside_project: |
| 52 | | print "Scrapy %s - project: %s\n" % (scrapy.__version__, \ |
| 53 | | settings['BOT_NAME']) |
| 54 | | else: |
| 55 | | print "Scrapy %s - no active project\n" % scrapy.__version__ |
| 56 | | print "Usage" |
| 57 | | print "=====\n" |
| 58 | | print "To run a command:" |
| 59 | | print " scrapy-ctl.py <command> [options] [args]\n" |
| 60 | | print "To get help:" |
| 61 | | print " scrapy-ctl.py <command> -h\n" |
| 62 | | print "Available commands" |
| 63 | | print "==================\n" |
| 64 | | cmds = _get_commands_dict() |
| 65 | | for cmdname, cmdclass in sorted(cmds.iteritems()): |
| 66 | | if inside_project or not cmdclass.requires_project: |
| 67 | | print "%s %s" % (cmdname, cmdclass.syntax()) |
| 68 | | print " %s" % cmdclass.short_desc() |
| 69 | | print |
| 70 | | |
| 71 | | def _update_default_settings(module, cmdname): |
| 72 | | if not module: |
| 73 | | return |
| 74 | | try: |
| 75 | | mod = __import__('%s.%s' % (module, cmdname), {}, {}, ['']) |
| 76 | | except ImportError: |
| 77 | | return |
| 78 | | settingsdict = vars(mod) |
| 79 | | for k, v in settingsdict.iteritems(): |
| 80 | | if not k.startswith("_"): |
| 81 | | settings.defaults[k] = v |
| 82 | | |
| 83 | | def execute(argv=None): |
| 84 | | if argv is None: |
| 85 | | argv = sys.argv |
| 86 | | |
| 87 | | cmds = _get_commands_dict() |
| 88 | | |
| 89 | | cmdname = _get_command_name(argv) |
| 90 | | _update_default_settings('scrapy.conf.commands', cmdname) |
| 91 | | _update_default_settings(settings['COMMANDS_SETTINGS_MODULE'], cmdname) |
| 92 | | |
| 93 | | parser = optparse.OptionParser(formatter=optparse.TitledHelpFormatter(), \ |
| 94 | | conflict_handler='resolve', add_help_option=False) |
| 95 | | |
| 96 | | if cmdname in cmds: |
| 97 | | cmd = cmds[cmdname] |
| 98 | | cmd.add_options(parser) |
| 99 | | opts, args = parser.parse_args(args=argv[1:]) |
| 100 | | cmd.process_options(args, opts) |
| 101 | | parser.usage = "%%prog %s %s" % (cmdname, cmd.syntax()) |
| 102 | | parser.description = cmd.long_desc() |
| 103 | | if cmd.requires_project and not settings.settings_module: |
| 104 | | print "Error running: scrapy-ctl.py %s\n" % cmdname |
| 105 | | print "Cannot find project settings module in python path: %s" % \ |
| 106 | | settings.settings_module_path |
| 107 | | sys.exit(1) |
| 108 | | if opts.help: |
| 109 | | parser.print_help() |
| 110 | | sys.exit() |
| 111 | | elif not cmdname: |
| 112 | | cmd = ScrapyCommand() |
| 113 | | cmd.add_options(parser) |
| 114 | | opts, args = parser.parse_args(args=argv) |
| 115 | | cmd.process_options(args, opts) |
| 116 | | _print_usage(settings.settings_module) |
| 117 | | sys.exit(2) |
| 118 | | else: |
| 119 | | print "Unknown command: %s\n" % cmdname |
| 120 | | print 'Use "scrapy-ctl.py -h" for help' |
| 121 | | sys.exit(2) |
| 122 | | |
| 123 | | del args[0] # remove command name from args |
| 124 | | send_catch_log(signal=command_executed, cmdname=cmdname, cmdobj=cmd, \ |
| 125 | | args=args, opts=opts) |
| 126 | | from scrapy.core.manager import scrapymanager |
| 127 | | scrapymanager.configure(control_reactor=True) |
| 128 | | ret = _run_command(cmd, args, opts) |
| 129 | | if ret is False: |
| 130 | | parser.print_help() |
| 131 | | |
| 132 | | def _run_command(cmd, args, opts): |
| 133 | | if opts.profile or opts.lsprof: |
| 134 | | return _run_command_profiled(cmd, args, opts) |
| 135 | | else: |
| 136 | | return cmd.run(args, opts) |
| 137 | | |
| 138 | | def _run_command_profiled(cmd, args, opts): |
| 139 | | if opts.profile: |
| 140 | | log.msg("writing cProfile stats to %r" % opts.profile) |
| 141 | | if opts.lsprof: |
| 142 | | log.msg("writing lsprof stats to %r" % opts.lsprof) |
| 143 | | loc = locals() |
| 144 | | p = cProfile.Profile() |
| 145 | | p.runctx('ret = cmd.run(args, opts)', globals(), loc) |
| 146 | | if opts.profile: |
| 147 | | p.dump_stats(opts.profile) |
| 148 | | k = lsprofcalltree.KCacheGrind(p) |
| 149 | | if opts.lsprof: |
| 150 | | with open(opts.lsprof, 'w') as f: |
| 151 | | k.output(f) |
| 152 | | return loc['ret'] |
| 153 | | |
| 154 | | if __name__ == '__main__': |
| 155 | | execute() |
| | 6 | def execute(): |
| | 7 | warnings.warn("scrapy.command.cmdline.execute() is deprecated, modify your " \ |
| | 8 | "project-ctl.py script to use scrapy.cmdline.execute() instead", \ |
| | 9 | DeprecationWarning, stacklevel=2) |
| | 10 | cmdline.execute() |
