swh.scheduler package

Subpackages

• swh.scheduler.api package
  • Submodules
    • swh.scheduler.api.client module
    • swh.scheduler.api.serializers module
    • swh.scheduler.api.server module
  • Module contents
• swh.scheduler.celery_backend package
  • Submodules
    • swh.scheduler.celery_backend.config module
    • swh.scheduler.celery_backend.first_visits module
    • swh.scheduler.celery_backend.pika_listener module
    • swh.scheduler.celery_backend.recurrent_visits module
    • swh.scheduler.celery_backend.runner module
    • swh.scheduler.celery_backend.utils module
  • Module contents
• swh.scheduler.cli package
  • Submodules
    • swh.scheduler.cli.add_forge_now module
    • swh.scheduler.cli.admin module
    • swh.scheduler.cli.celery_monitor module
    • swh.scheduler.cli.config module
    • swh.scheduler.cli.journal module
    • swh.scheduler.cli.origin module
    • swh.scheduler.cli.origin_utils module
    • swh.scheduler.cli.simulator module
    • swh.scheduler.cli.task module
    • swh.scheduler.cli.task_type module
    • swh.scheduler.cli.utils module
  • Module contents
    • main()
• swh.scheduler.simulator package
  • Submodules
    • swh.scheduler.simulator.common module
    • swh.scheduler.simulator.origin_scheduler module
    • swh.scheduler.simulator.origins module
    • swh.scheduler.simulator.task_scheduler module
  • Module contents
    • update_metrics_process()
    • worker_process()
    • setup()
    • fill_test_data()
    • run()
• swh.scheduler.sql package
  • Subpackages
    • swh.scheduler.sql.upgrades package
  • Module contents

Submodules

• swh.scheduler.backend module
  • ensure_register_last_visit_status()
  • execute_values_generator()
  • jsonize()
  • format_query()
  • mutate_task_dict()
  • SchedulerBackend
    • SchedulerBackend.current_version
    • SchedulerBackend.get_db()
    • SchedulerBackend.put_db()
    • SchedulerBackend.task_type_keys
    • SchedulerBackend.create_task_type()
    • SchedulerBackend.get_task_type()
    • SchedulerBackend.get_task_types()
    • SchedulerBackend.get_listers()
    • SchedulerBackend.get_listers_by_id()
    • SchedulerBackend.get_lister()
    • SchedulerBackend.get_or_create_lister()
    • SchedulerBackend.update_lister()
    • SchedulerBackend.record_listed_origins()
    • SchedulerBackend.get_listed_origins()
    • SchedulerBackend.get_visit_types_for_listed_origins()
    • SchedulerBackend.grab_next_visits()
    • SchedulerBackend.task_keys
    • SchedulerBackend.task_create_keys
    • SchedulerBackend.create_tasks()
    • SchedulerBackend.set_status_tasks()
    • SchedulerBackend.disable_tasks()
    • SchedulerBackend.search_tasks()
    • SchedulerBackend.get_tasks()
    • SchedulerBackend.peek_ready_tasks()
    • SchedulerBackend.grab_ready_tasks()
    • SchedulerBackend.peek_ready_priority_tasks()
    • SchedulerBackend.grab_ready_priority_tasks()
    • SchedulerBackend.task_run_create_keys
    • SchedulerBackend.schedule_task_run()
    • SchedulerBackend.mass_schedule_task_runs()
    • SchedulerBackend.start_task_run()
    • SchedulerBackend.end_task_run()
    • SchedulerBackend.filter_task_to_archive()
    • SchedulerBackend.delete_archived_tasks()
    • SchedulerBackend.task_run_keys
    • SchedulerBackend.get_task_runs()
    • SchedulerBackend.origin_visit_stats_upsert()
    • SchedulerBackend.origin_visit_stats_get()
    • SchedulerBackend.visit_scheduler_queue_position_get()
    • SchedulerBackend.visit_scheduler_queue_position_set()
    • SchedulerBackend.update_metrics()
    • SchedulerBackend.get_metrics()
• swh.scheduler.cli_utils module
• swh.scheduler.exc module
  • SchedulerException
  • StaleData
  • UnknownPolicy
• swh.scheduler.in_memory module
  • InMemoryScheduler
    • InMemoryScheduler.create_task_type()
    • InMemoryScheduler.get_task_type()
    • InMemoryScheduler.get_task_types()
    • InMemoryScheduler.get_listers()
    • InMemoryScheduler.get_listers_by_id()
    • InMemoryScheduler.get_lister()
    • InMemoryScheduler.get_or_create_lister()
    • InMemoryScheduler.update_lister()
    • InMemoryScheduler.record_listed_origins()
    • InMemoryScheduler.get_listed_origins()
    • InMemoryScheduler.get_visit_types_for_listed_origins()
    • InMemoryScheduler.grab_next_visits()
    • InMemoryScheduler.create_tasks()
    • InMemoryScheduler.set_status_tasks()
    • InMemoryScheduler.disable_tasks()
    • InMemoryScheduler.search_tasks()
    • InMemoryScheduler.get_tasks()
    • InMemoryScheduler.peek_ready_tasks()
    • InMemoryScheduler.grab_ready_tasks()
    • InMemoryScheduler.peek_ready_priority_tasks()
    • InMemoryScheduler.grab_ready_priority_tasks()
    • InMemoryScheduler.schedule_task_run()
    • InMemoryScheduler.mass_schedule_task_runs()
    • InMemoryScheduler.start_task_run()
    • InMemoryScheduler.end_task_run()
    • InMemoryScheduler.filter_task_to_archive()
    • InMemoryScheduler.delete_archived_tasks()
    • InMemoryScheduler.get_task_runs()
    • InMemoryScheduler.origin_visit_stats_upsert()
    • InMemoryScheduler.origin_visit_stats_get()
    • InMemoryScheduler.visit_scheduler_queue_position_get()
    • InMemoryScheduler.visit_scheduler_queue_position_set()
    • InMemoryScheduler.update_metrics()
    • InMemoryScheduler.get_metrics()
• swh.scheduler.interface module
  • PaginatedListedOriginList
  • SchedulerInterface
    • SchedulerInterface.create_task_type()
    • SchedulerInterface.get_task_type()
    • SchedulerInterface.get_task_types()
    • SchedulerInterface.create_tasks()
    • SchedulerInterface.set_status_tasks()
    • SchedulerInterface.disable_tasks()
    • SchedulerInterface.search_tasks()
    • SchedulerInterface.get_tasks()
    • SchedulerInterface.peek_ready_tasks()
    • SchedulerInterface.grab_ready_tasks()
    • SchedulerInterface.peek_ready_priority_tasks()
    • SchedulerInterface.grab_ready_priority_tasks()
    • SchedulerInterface.schedule_task_run()
    • SchedulerInterface.mass_schedule_task_runs()
    • SchedulerInterface.start_task_run()
    • SchedulerInterface.end_task_run()
    • SchedulerInterface.filter_task_to_archive()
    • SchedulerInterface.delete_archived_tasks()
    • SchedulerInterface.get_task_runs()
    • SchedulerInterface.get_listers()
    • SchedulerInterface.get_listers_by_id()
    • SchedulerInterface.get_lister()
    • SchedulerInterface.get_or_create_lister()
    • SchedulerInterface.update_lister()
    • SchedulerInterface.record_listed_origins()
    • SchedulerInterface.get_listed_origins()
    • SchedulerInterface.get_visit_types_for_listed_origins()
    • SchedulerInterface.grab_next_visits()
    • SchedulerInterface.origin_visit_stats_upsert()
    • SchedulerInterface.origin_visit_stats_get()
    • SchedulerInterface.visit_scheduler_queue_position_get()
    • SchedulerInterface.visit_scheduler_queue_position_set()
    • SchedulerInterface.update_metrics()
    • SchedulerInterface.get_metrics()
• swh.scheduler.journal_client module
  • DISABLE_ORIGIN_THRESHOLD
  • MAX_NEXT_POSITION_OFFSET
  • max_date()
  • from_position_offset_to_days()
  • next_visit_queue_position()
  • get_last_status()
  • process_journal_objects()
• swh.scheduler.model module
  • check_timestamptz()
  • BaseSchedulerModel
    • BaseSchedulerModel.primary_key_columns()
    • BaseSchedulerModel.select_columns()
    • BaseSchedulerModel.insert_columns_and_metavars()
    • BaseSchedulerModel.evolve()
    • BaseSchedulerModel.to_dict()
    • BaseSchedulerModel.to_tuple()
  • Lister
  • ListedOrigin
  • LastVisitStatus
    • LastVisitStatus.successful
    • LastVisitStatus.failed
    • LastVisitStatus.not_found
  • convert_last_visit_status()
  • OriginVisitStats
    • OriginVisitStats.url
    • OriginVisitStats.visit_type
    • OriginVisitStats.last_successful
    • OriginVisitStats.last_visit
    • OriginVisitStats.last_visit_status
    • OriginVisitStats.last_scheduled
    • OriginVisitStats.last_snapshot
    • OriginVisitStats.next_visit_queue_position
    • OriginVisitStats.next_position_offset
    • OriginVisitStats.successive_visits
    • OriginVisitStats.check_last_successful()
    • OriginVisitStats.check_last_visit()
  • SchedulerMetrics
    • SchedulerMetrics.lister_id
    • SchedulerMetrics.visit_type
    • SchedulerMetrics.last_update
    • SchedulerMetrics.origins_known
    • SchedulerMetrics.origins_enabled
    • SchedulerMetrics.origins_never_visited
    • SchedulerMetrics.origins_with_pending_changes
  • TaskType
    • TaskType.type
    • TaskType.description
    • TaskType.backend_name
    • TaskType.default_interval
    • TaskType.min_interval
    • TaskType.max_interval
    • TaskType.backoff_factor
    • TaskType.max_queue_length
    • TaskType.num_retries
    • TaskType.retry_delay
  • TaskArguments
    • TaskArguments.args
    • TaskArguments.kwargs
  • Task
    • Task.type
    • Task.arguments
    • Task.next_run
    • Task.status
    • Task.policy
    • Task.retries_left
    • Task.id
    • Task.current_interval
    • Task.priority
  • TaskRun
    • TaskRun.task
    • TaskRun.id
    • TaskRun.backend_id
    • TaskRun.scheduled
    • TaskRun.started
    • TaskRun.ended
    • TaskRun.metadata
    • TaskRun.status
• swh.scheduler.pytest_plugin module
  • swh_scheduler_class()
  • swh_scheduler_config()
  • swh_scheduler()
  • swh_db_scheduler()
  • swh_scheduler_celery_app()
  • swh_scheduler_celery_includes()
  • swh_scheduler_celery_worker()
• swh.scheduler.task module
  • ts()
  • SWHTask
    • SWHTask.reject_on_worker_lost
    • SWHTask.statsd
    • SWHTask.on_failure()
    • SWHTask.on_success()
    • SWHTask.log
    • SWHTask.ignore_result
    • SWHTask.priority
    • SWHTask.rate_limit
    • SWHTask.request_stack
    • SWHTask.run()
    • SWHTask.serializer
    • SWHTask.store_errors_even_if_ignored
    • SWHTask.track_started
    • SWHTask.typing
• swh.scheduler.utils module
  • utcnow()
  • get_task()
  • create_task()
  • create_origin_task()
  • create_origin_tasks()
  • create_oneshot_task()

Module contents

swh.scheduler.get_scheduler(cls: str, **kwargs) → SchedulerInterface

Get a scheduler object of class cls with arguments **kwargs.

Parameters:

• cls – scheduler’s class

• kwargs – arguments to pass to the class’ constructor

Returns:

an instance of swh.scheduler.

Raises:

ValueError if passed an unknown storage class. –