apache_beam.runners package

Runner objects execute a Pipeline.

This package defines runners, which are used to execute a pipeline.

Subpackages

• apache_beam.runners.dask package
  • Submodules
    • apache_beam.runners.dask.dask_runner module
      • DaskOptions
      • DaskRunnerResult
      • DaskRunner
    • apache_beam.runners.dask.overrides module
      • dask_overrides()
• apache_beam.runners.dataflow package
  • Submodules
    • apache_beam.runners.dataflow.dataflow_exercise_metrics_pipeline module
      • metric_matchers()
      • UserMetricsDoFn
      • apply_and_run()
    • apache_beam.runners.dataflow.dataflow_exercise_streaming_metrics_pipeline module
      • StreamingUserMetricsDoFn
      • run()
    • apache_beam.runners.dataflow.dataflow_job_service module
      • DataflowBeamJob
      • run()
    • apache_beam.runners.dataflow.dataflow_metrics module
      • DataflowMetrics
      • main()
    • apache_beam.runners.dataflow.dataflow_runner module
      • DataflowRunner
    • apache_beam.runners.dataflow.ptransform_overrides module
      • NativeReadPTransformOverride
    • apache_beam.runners.dataflow.test_dataflow_runner module
      • TestDataflowRunner
• apache_beam.runners.direct package
  • Submodules
    • apache_beam.runners.direct.bundle_factory module
      • BundleFactory
    • apache_beam.runners.direct.clock module
      • Clock
      • RealClock
      • TestClock
    • apache_beam.runners.direct.consumer_tracking_pipeline_visitor module
      • ConsumerTrackingPipelineVisitor
    • apache_beam.runners.direct.direct_metrics module
      • MetricAggregator
      • CounterAggregator
      • GenericAggregator
      • DirectMetrics
      • DirectMetric
    • apache_beam.runners.direct.direct_runner module
      • BundleBasedDirectRunner
      • DirectRunner
      • SwitchingDirectRunner
    • apache_beam.runners.direct.direct_userstate module
      • DirectRuntimeState
      • ReadModifyWriteRuntimeState
      • BagRuntimeState
      • SetRuntimeState
      • CombiningValueRuntimeState
      • DirectUserStateContext
    • apache_beam.runners.direct.evaluation_context module
      • EvaluationContext
      • DirectUnmergedState
      • DirectStepContext
    • apache_beam.runners.direct.executor module
      • TransformExecutor
      • Executor
    • apache_beam.runners.direct.sdf_direct_runner module
      • SplittableParDoOverride
      • SplittableParDo
      • ElementAndRestriction
      • PairWithRestrictionFn
      • SplitRestrictionFn
      • ExplodeWindowsFn
      • RandomUniqueKeyFn
      • ProcessKeyedElements
      • ProcessKeyedElementsViaKeyedWorkItemsOverride
      • ProcessKeyedElementsViaKeyedWorkItems
      • ProcessElements
      • ProcessFn
      • SDFProcessElementInvoker
    • apache_beam.runners.direct.test_direct_runner module
      • TestDirectRunner
    • apache_beam.runners.direct.test_stream_impl module
    • apache_beam.runners.direct.transform_evaluator module
      • TransformEvaluatorRegistry
      • RootBundleProvider
      • DefaultRootBundleProvider
    • apache_beam.runners.direct.util module
      • TransformResult
      • TimerFiring
      • KeyedWorkItem
    • apache_beam.runners.direct.watermark_manager module
      • WatermarkManager
• apache_beam.runners.interactive package
  • Subpackages
    • apache_beam.runners.interactive.caching package
      • Submodules
    • apache_beam.runners.interactive.dataproc package
      • Submodules
    • apache_beam.runners.interactive.display package
      • Submodules
    • apache_beam.runners.interactive.messaging package
      • Submodules
    • apache_beam.runners.interactive.options package
      • Submodules
    • apache_beam.runners.interactive.sql package
      • Submodules
    • apache_beam.runners.interactive.testing package
      • Subpackages
      • Submodules
  • Submodules
    • apache_beam.runners.interactive.augmented_pipeline module
      • AugmentedPipeline
    • apache_beam.runners.interactive.background_caching_job module
      • BackgroundCachingJob
      • attempt_to_run_background_caching_job()
      • is_background_caching_job_needed()
      • is_cache_complete()
      • has_source_to_cache()
      • attempt_to_cancel_background_caching_job()
      • attempt_to_stop_test_stream_service()
      • is_a_test_stream_service_running()
      • is_source_to_cache_changed()
      • extract_source_to_cache_signature()
    • apache_beam.runners.interactive.cache_manager module
      • CacheManager
      • FileBasedCacheManager
      • ReadCache
      • WriteCache
      • SafeFastPrimitivesCoder
      • Base64Coder
    • apache_beam.runners.interactive.interactive_beam module
      • Options
      • Recordings
      • Clusters
      • watch()
      • show()
      • collect()
      • show_graph()
      • evict_recorded_data()
    • apache_beam.runners.interactive.interactive_environment module
      • current_env()
      • new_env()
      • InteractiveEnvironment
    • apache_beam.runners.interactive.interactive_runner module
      • InteractiveRunner
      • PipelineResult
    • apache_beam.runners.interactive.pipeline_fragment module
      • PipelineFragment
      • should_skip_pruning()
    • apache_beam.runners.interactive.pipeline_instrument module
      • PipelineInstrument
      • build_pipeline_instrument()
      • pcoll_to_pcoll_id()
    • apache_beam.runners.interactive.recording_manager module
      • ElementStream
      • Recording
      • RecordingManager
    • apache_beam.runners.interactive.user_pipeline_tracker module
      • UserPipelineTracker
    • apache_beam.runners.interactive.utils module
      • to_element_list()
      • elements_to_df()
      • register_ipython_log_handler()
      • IPythonLogHandler
      • obfuscate()
      • ProgressIndicator
      • progress_indicated()
      • as_json()
      • deferred_df_to_pcollection()
      • pcoll_by_name()
      • find_pcoll_name()
      • cacheables()
      • watch_sources()
      • has_unbounded_sources()
      • unbounded_sources()
      • create_var_in_main()
      • assert_bucket_exists()
      • detect_pipeline_runner()
• apache_beam.runners.job package
  • Submodules
    • apache_beam.runners.job.manager module
      • DockerRPCManager
    • apache_beam.runners.job.utils module
      • dict_to_struct()
      • struct_to_dict()

Submodules

• apache_beam.runners.pipeline_context module
  • PortableObject
    • PortableObject.to_runner_api()
    • PortableObject.from_runner_api()
  • PipelineContext
    • PipelineContext.add_requirement()
    • PipelineContext.requirements()
    • PipelineContext.coder_id_from_element_type()
    • PipelineContext.deterministic_coder()
    • PipelineContext.element_type_from_coder_id()
    • PipelineContext.from_runner_api()
    • PipelineContext.to_runner_api()
    • PipelineContext.default_environment_id()
    • PipelineContext.get_environment_id_for_resource_hints()
• apache_beam.runners.pipeline_utils module
  • group_by_key_input_visitor()
  • validate_pipeline_graph()
  • merge_common_environments()
  • merge_superset_dep_environments()
  • update_environments()
• apache_beam.runners.render module
  • RenderOptions
  • PipelineRenderer
    • PipelineRenderer.update()
    • PipelineRenderer.style()
    • PipelineRenderer.to_dot()
    • PipelineRenderer.to_dot_iter()
    • PipelineRenderer.transform_to_dot()
    • PipelineRenderer.transform_node()
    • PipelineRenderer.transform_attributes()
    • PipelineRenderer.pcoll_leaf_consumers_iter()
    • PipelineRenderer.pcoll_leaf_consumers()
    • PipelineRenderer.is_leaf()
    • PipelineRenderer.info()
    • PipelineRenderer.layout_dot()
    • PipelineRenderer.page_callback_data()
    • PipelineRenderer.render_data()
    • PipelineRenderer.render_json()
    • PipelineRenderer.page()
  • RenderRunner
    • RenderRunner.run_pipeline()
    • RenderRunner.run_portable_pipeline()
  • RenderPipelineResult
    • RenderPipelineResult.wait_until_finish()
    • RenderPipelineResult.monitoring_infos()
  • run()
  • render_one()
  • run_server()
• apache_beam.runners.runner module
  • PipelineRunner
    • PipelineRunner.run()
    • PipelineRunner.run_async()
    • PipelineRunner.run_portable_pipeline()
    • PipelineRunner.default_environment()
    • PipelineRunner.run_pipeline()
    • PipelineRunner.apply()
    • PipelineRunner.apply_PTransform()
    • PipelineRunner.is_fnapi_compatible()
    • PipelineRunner.check_requirements()
    • PipelineRunner.default_pickle_library_override()
  • PipelineState
    • PipelineState.UNKNOWN
    • PipelineState.STARTING
    • PipelineState.STOPPED
    • PipelineState.RUNNING
    • PipelineState.DONE
    • PipelineState.FAILED
    • PipelineState.CANCELLED
    • PipelineState.UPDATED
    • PipelineState.DRAINING
    • PipelineState.DRAINED
    • PipelineState.PENDING
    • PipelineState.CANCELLING
    • PipelineState.RESOURCE_CLEANING_UP
    • PipelineState.UNRECOGNIZED
    • PipelineState.is_terminal()
  • PipelineResult
    • PipelineResult.state
    • PipelineResult.wait_until_finish()
    • PipelineResult.cancel()
    • PipelineResult.metrics()
    • PipelineResult.aggregated_values()
• apache_beam.runners.sdf_utils module
  • SplitResultPrimary
    • SplitResultPrimary.primary_value
  • SplitResultResidual
    • SplitResultResidual.current_watermark
    • SplitResultResidual.deferred_timestamp
    • SplitResultResidual.residual_value
  • ThreadsafeRestrictionTracker
    • ThreadsafeRestrictionTracker.current_restriction()
    • ThreadsafeRestrictionTracker.try_claim()
    • ThreadsafeRestrictionTracker.defer_remainder()
    • ThreadsafeRestrictionTracker.check_done()
    • ThreadsafeRestrictionTracker.current_progress()
    • ThreadsafeRestrictionTracker.try_split()
    • ThreadsafeRestrictionTracker.deferred_status()
    • ThreadsafeRestrictionTracker.is_bounded()
  • RestrictionTrackerView
    • RestrictionTrackerView.current_restriction()
    • RestrictionTrackerView.try_claim()
    • RestrictionTrackerView.defer_remainder()
    • RestrictionTrackerView.is_bounded()
  • ThreadsafeWatermarkEstimator
    • ThreadsafeWatermarkEstimator.get_estimator_state()
    • ThreadsafeWatermarkEstimator.current_watermark()
    • ThreadsafeWatermarkEstimator.observe_timestamp()
  • NoOpWatermarkEstimatorProvider
    • NoOpWatermarkEstimatorProvider.initial_estimator_state()
    • NoOpWatermarkEstimatorProvider.create_watermark_estimator()
• apache_beam.runners.trivial_runner module
  • TrivialRunner
    • TrivialRunner.run_portable_pipeline()
    • TrivialRunner.execute_transform()
    • TrivialRunner.execute_executable_stage()
    • TrivialRunner.group_by_key_and_window()
    • TrivialRunner.supported_requirements()
  • ExecutionState
    • ExecutionState.register_process_bundle_descriptor()
    • ExecutionState.get_pcollection_contents()
    • ExecutionState.set_pcollection_contents()
    • ExecutionState.new_id()
    • ExecutionState.windowed_coder()
    • ExecutionState.windowing_strategy()
    • ExecutionState.windowed_coder_id()
  • is_primitive_transform()
  • only_element()
  • decode_all()
  • encode_all()