mex.backend package

Subpackages

• mex.backend.auxiliary package
  • Submodules
  • mex.backend.auxiliary.ldap module
    • search_persons_or_contact_points_in_ldap()
  • mex.backend.auxiliary.orcid module
    • search_persons_in_orcid()
  • mex.backend.auxiliary.organigram module
    • extracted_organizational_units()
  • mex.backend.auxiliary.primary_source module
    • _fetch_or_insert_primary_source()
    • extracted_primary_source_ldap()
    • extracted_primary_source_orcid()
    • extracted_primary_source_organigram()
    • extracted_primary_source_wikidata()
  • mex.backend.auxiliary.utils module
    • fetch_extracted_item_by_source_identifiers()
  • mex.backend.auxiliary.wikidata module
    • _fetch_or_insert_organization()
    • extracted_organization_rki()
    • search_organizations_in_wikidata()
  • Module contents
• mex.backend.cache package
  • Submodules
  • mex.backend.cache.connector module
    • CacheConnector
      • CacheConnector.__init__()
      • CacheConnector.close()
      • CacheConnector.delete_value()
      • CacheConnector.flush()
      • CacheConnector.get_value()
      • CacheConnector.metrics()
      • CacheConnector.set_value()
    • LocalCache
      • LocalCache.close()
      • LocalCache.delete()
      • LocalCache.flushdb()
      • LocalCache.info()
      • LocalCache.set()
    • ValkeyCache
      • ValkeyCache.__init__()
      • ValkeyCache.close()
      • ValkeyCache.delete()
      • ValkeyCache.flushdb()
      • ValkeyCache.get()
      • ValkeyCache.info()
      • ValkeyCache.set()
  • Module contents
• mex.backend.extracted package
  • Submodules
  • mex.backend.extracted.helpers module
    • get_extracted_item_from_graph()
    • get_extracted_items_from_graph()
    • search_extracted_items_in_graph()
  • mex.backend.extracted.main module
    • get_extracted_item()
    • search_extracted_items()
    • search_extracted_items_advanced()
  • Module contents
• mex.backend.graph package
  • Submodules
  • mex.backend.graph.connector module
    • GraphConnector
      • GraphConnector.__init__()
      • GraphConnector._check_connectivity_and_authentication()
      • GraphConnector._check_match_preconditions_tx()
      • GraphConnector._fetch_extracted_or_rule_items()
      • GraphConnector._init_driver()
      • GraphConnector._match_item_tx()
      • GraphConnector._run_ingest_in_transaction()
      • GraphConnector._seed_constraints()
      • GraphConnector._seed_data()
      • GraphConnector._seed_indices()
      • GraphConnector.close()
      • GraphConnector.commit()
      • GraphConnector.delete_item()
      • GraphConnector.delete_rule_set()
      • GraphConnector.exists_item()
      • GraphConnector.fetch_extracted_items()
      • GraphConnector.fetch_identities()
      • GraphConnector.fetch_merged_items()
      • GraphConnector.fetch_rule_items()
      • GraphConnector.flush()
      • GraphConnector.ingest_items()
      • GraphConnector.match_item()
  • mex.backend.graph.constants module
  • mex.backend.graph.exceptions module
    • DeletionFailedError
    • GraphError
      • GraphError.__init__()
      • GraphError.errors()
      • GraphError.is_retryable()
    • InconsistentGraphError
    • IngestionError
    • MatchingError
    • MultipleResultsFoundError
    • NoResultFoundError
  • mex.backend.graph.models module
    • EdgeExporter
      • EdgeExporter.transform()
    • ExtractedPrimarySourceWithHardcodedIdentifiers
      • ExtractedPrimarySourceWithHardcodedIdentifiers.entityType
      • ExtractedPrimarySourceWithHardcodedIdentifiers.identifier
      • ExtractedPrimarySourceWithHardcodedIdentifiers.model_config
      • ExtractedPrimarySourceWithHardcodedIdentifiers.stableTargetId
    • GraphRel
      • GraphRel.edgeLabel
      • GraphRel.edgeProps
      • GraphRel.nodeLabels
      • GraphRel.nodeProps
    • IngestData
      • IngestData.createRels
      • IngestData.entityType
      • IngestData.identifier
      • IngestData.linkRels
      • IngestData.metadata()
      • IngestData.model_config
      • IngestData.nodeProps
      • IngestData.sort_create_rels()
      • IngestData.sort_link_rels()
      • IngestData.stableTargetId
    • IngestParams
      • IngestParams.delete_node_edges
      • IngestParams.detach_node_edges
      • IngestParams.has_create_rels
      • IngestParams.has_link_rels
      • IngestParams.merged_label
      • IngestParams.model_config
      • IngestParams.node_label
      • IngestParams.used_nested_labels
      • IngestParams.used_referenced_labels
    • RawReferenceFilter
      • RawReferenceFilter.field
      • RawReferenceFilter.identifiers
    • Result
      • Result.__init__()
      • Result.all()
      • Result.get_update_counters()
      • Result.log_notifications()
      • Result.one()
      • Result.one_or_none()
  • mex.backend.graph.query module
    • Query
      • Query.__init__()
      • Query.render()
    • QueryBuilder
      • QueryBuilder.__init__()
      • QueryBuilder._get_ingest_query_for_entity_type()
      • QueryBuilder.close()
    • render_constraints()
  • mex.backend.graph.transform module
    • _SearchResultReference
      • _SearchResultReference.label
      • _SearchResultReference.position
      • _SearchResultReference.value
    • expand_references_in_search_result()
    • get_error_details_from_neo4j_error()
    • get_graph_rel_id()
    • transform_model_into_ingest_data()
    • transform_reference_filters_to_raw_fields()
    • transform_reference_filters_to_raw_filters()
    • validate_ingested_data()
  • Module contents
• mex.backend.http_test_server package
  • Submodules
  • mex.backend.http_test_server.main module
    • head_http_test_server()
    • http_test_server()
    • lifespan()
    • post_datscha_web_login()
  • mex.backend.http_test_server.settings module
    • HttpTestServerSettings
      • HttpTestServerSettings.http_test_server_host
      • HttpTestServerSettings.http_test_server_port
      • HttpTestServerSettings.http_test_server_root_path
      • HttpTestServerSettings.http_test_server_test_data_directory
      • HttpTestServerSettings.model_config
  • Module contents
• mex.backend.identity package
  • Submodules
  • mex.backend.identity.main module
    • assign_identity()
    • fetch_identity()
  • mex.backend.identity.provider module
    • GraphIdentityProvider
      • GraphIdentityProvider.__init__()
      • GraphIdentityProvider.assign()
      • GraphIdentityProvider.close()
      • GraphIdentityProvider.fetch()
      • GraphIdentityProvider.metrics()
  • Module contents
• mex.backend.ingest package
  • Submodules
  • mex.backend.ingest.main module
    • ingest_items()
  • Module contents
• mex.backend.ldap package
  • Submodules
  • mex.backend.ldap.main module
    • get_merged_person_from_login()
  • Module contents
• mex.backend.match package
  • Submodules
  • mex.backend.match.helpers module
    • match_item_in_graph()
  • mex.backend.match.main module
    • match_item()
  • Module contents
• mex.backend.merged package
  • Submodules
  • mex.backend.merged.helpers module
    • delete_merged_item_from_graph()
    • get_merged_item_from_graph()
    • merge_search_result_item()
    • search_merged_items_in_graph()
  • mex.backend.merged.main module
    • delete_merged_item()
    • get_merged_item()
    • search_merged_items()
    • search_merged_items_advanced()
  • Module contents
• mex.backend.preview package
  • Submodules
  • mex.backend.preview.main module
    • get_preview_item()
    • search_preview_items()
    • search_preview_items_advanced()
  • Module contents
• mex.backend.publishable_merged package
  • Submodules
  • mex.backend.publishable_merged.helpers module
    • merge_publishable_search_result_item()
    • search_publishable_merged_items_in_graph()
  • mex.backend.publishable_merged.main module
    • search_publishable_merged_items()
    • search_publishable_merged_items_advanced()
  • Module contents
• mex.backend.rules package
  • Submodules
  • mex.backend.rules.helpers module
    • create_and_get_rule_set()
    • delete_rule_set_from_graph()
    • get_rule_set_from_graph()
    • update_and_get_rule_set()
  • mex.backend.rules.main module
    • create_rule_set()
    • delete_rule_set()
    • get_rule_set()
    • update_rule_set()
  • mex.backend.rules.transform module
    • transform_raw_rules_to_rule_set_response()
  • Module contents
• mex.backend.seed package
  • Submodules
  • mex.backend.seed.main module
  • Module contents
• mex.backend.system package
  • Submodules
  • mex.backend.system.main module
    • check_system_status()
    • get_prometheus_metrics()
  • mex.backend.system.models module
    • GraphStatus
      • GraphStatus.model_config
      • GraphStatus.status
    • SystemStatus
      • SystemStatus.model_config
      • SystemStatus.status
      • SystemStatus.version
  • Module contents
• mex.backend.testing package
  • Submodules
  • mex.backend.testing.ldap module
    • get_merged_person_from_login()
    • search_persons_or_contact_points_in_ldap()
  • mex.backend.testing.main module
  • mex.backend.testing.security module
    • is_ldap_authenticated_mocked()
  • mex.backend.testing.system module
    • flush_graph_database()
  • Module contents

Submodules

mex.backend.exceptions module

exception mex.backend.exceptions.BackendError

Bases: MExError

Base backend error that offer details on underlying pydantic errors.

errors() → list[ErrorDetails]

Details about underlying pydantic errors.

is_retryable() → bool

Whether the error is retryable.

class mex.backend.exceptions.DebuggingInfo(*, errors: list[dict[str, Any]], scope: DebuggingScope)

Bases: BaseModel

Debugging information for error responses.

errors: list[dict[str, Any]]

model_config = {}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

scope: DebuggingScope

class mex.backend.exceptions.DebuggingScope(*, http_version: str, method: str, path: str, path_params: dict[str, Any], query_string: str, scheme: str)

Bases: BaseModel

Scope for debugging info of error responses.

http_version: str

method: str

model_config = {'extra': 'ignore'}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

path: str

path_params: dict[str, Any]

query_string: str

scheme: str

class mex.backend.exceptions.DetailedError(*args, **kwargs)

Bases: Protocol

Protocol for errors that offer details.

errors() → list[ErrorDetails]

Details about each underlying error.

class mex.backend.exceptions.ErrorResponse(*, message: str, debug: DebuggingInfo)

Bases: BaseModel

Response model for user and system errors.

debug: DebuggingInfo

message: str

model_config = {}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

mex.backend.exceptions.handle_detailed_error(request: Request, exc: Exception) → Response

Handle detailed errors and provide debugging info.

mex.backend.exceptions.handle_uncaught_exception(request: Request, exc: Exception) → Response

Handle uncaught errors and provide debugging info.

mex.backend.helpers module

mex.backend.helpers.build_reference_filters(reference_field: ReferenceFieldName | None = None, referenced_identifier: Sequence[Identifier] | None = None, stable_target_id: Identifier | None = None) → list[ReferenceFilter]

Convert deprecated reference parameters to ReferenceFilter objects.

Parameters:

• reference_field – Deprecated query parameter for reference field name

• referenced_identifier – Deprecated query parameter for identifiers

• stable_target_id – Deprecated query parameter for stableTargetId

Raises:

HTTPException – If only one of the two parameters is provided

Returns:

list with a single ReferenceFilter, or empty list if neither parameter provided

mex.backend.logging module

mex.backend.main module

mex.backend.main.lifespan(_: FastAPI) → AsyncIterator[None]

Async context manager to execute startup and teardown of the FastAPI app.

mex.backend.migration module

mex.backend.migration.add_workflow_rule_to_all_rule_sets_on_db() → None

Add workflow rule to all rule sets in database.

mex.backend.migration.get_all_merged_item_results(connector: GraphConnector) → Generator[__annotationlib_name_1__]

Get all merged items from database.

mex.backend.migration.get_all_raw_rules(connector: GraphConnector) → Generator[__annotationlib_name_1__]

Get all raw rules in database.

mex.backend.migration.migrate() → None

Run migrations.

mex.backend.migration.transform_three_raw_rules_to_rule_set_response(raw_rules: Sequence[__annotationlib_name_1__]) → AccessPlatformRuleSetResponse | ActivityRuleSetResponse | BibliographicResourceRuleSetResponse | ConsentRuleSetResponse | ContactPointRuleSetResponse | DistributionRuleSetResponse | OrganizationRuleSetResponse | OrganizationalUnitRuleSetResponse | PersonRuleSetResponse | PrimarySourceRuleSetResponse | ResourceRuleSetResponse | VariableRuleSetResponse | VariableGroupRuleSetResponse

Transform a set of plain rules into a rule set response.

mex.backend.models module

class mex.backend.models.APIKeyDatabase(*, read: list[APIKey] = [], write: list[APIKey] = [])

Bases: BaseModel

A lookup from access level to list of allowed APIKeys.

model_config = {}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

read: list[APIKey]

write: list[APIKey]

class mex.backend.models.ReferenceFilter(*, field: ReferenceFieldName, identifiers: Annotated[list[Identifier | None], MinLen(min_length=1), MaxLen(max_length=100)])

Bases: BaseModel

Reference filter definition with a field and list of identifiers.

field: ReferenceFieldName

identifiers: Annotated[list[Identifier | None], FieldInfo(annotation=NoneType, required=True, metadata=[MinLen(min_length=1), MaxLen(max_length=100)])]

model_config = {}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

mex.backend.security module

mex.backend.security.has_read_access(api_key: ~typing.Annotated[str, ~fastapi.params.Depends(dependency=<fastapi.security.api_key.APIKeyHeader object at 0x7f2511cc2e40>, use_cache=True, scope=None)]) → None

Verify if api key has read access or write access.

Raises:

HTTPException if APIKey has no read access. –

Parameters:

api_key – the API key

mex.backend.security.has_write_access(api_key: ~typing.Annotated[str, ~fastapi.params.Depends(dependency=<fastapi.security.api_key.APIKeyHeader object at 0x7f2511cc2e40>, use_cache=True, scope=None)]) → None

Verify if provided api key has write access.

Raises:

HTTPException if APIKey has no write access. –

Parameters:

api_key – the API key

mex.backend.security.is_ldap_authenticated(credentials: ~typing.Annotated[~fastapi.security.http.HTTPBasicCredentials, ~fastapi.params.Depends(dependency=<fastapi.security.http.HTTPBasic object at 0x7f2511cc2f90>, use_cache=True, scope=None)]) → str

Authenticate against LDAP.

Raises:

HTTPException if credentials have no LDAP write access. –

Parameters:

credentials – username and password

Returns:

username

mex.backend.settings module

class mex.backend.settings.BackendSettings(_env_file: Path | str | Sequence[Path | str] | None = PosixPath('.'), _env_file_encoding: str | None = None, _env_nested_delimiter: str | None = None, _secrets_dir: str | Path | None = None, *, MEX_DEBUG: bool = False, MEX_SINK: list[Sink] = [Sink.NDJSON], MEX_ASSETS_DIR: Path = PosixPath('/home/runner/work/mex-backend/mex-backend/assets'), MEX_OPS_DIR: Path = PosixPath('/home/runner/work/mex-backend/mex-backend/ops'), MEX_WORK_DIR: Path = PosixPath('/home/runner/work/mex-backend/mex-backend'), MEX_IDENTITY_PROVIDER: IdentityProvider = IdentityProvider.MEMORY, MEX_BACKEND_API_URL: HttpUrl = HttpUrl('http://localhost:8080/'), MEX_BACKEND_API_KEY: SecretStr = SecretStr('**********'), MEX_BACKEND_API_PARALLELIZATION: int = 1, MEX_BACKEND_API_CHUNK_SIZE: int = 25, MEX_VERIFY_SESSION: bool | OpsPath = True, MEX_ORGANIGRAM_PATH: AssetsPath = AssetsPath('raw-data/organigram/organizational_units.json'), MEX_PRIMARY_SOURCES_PATH: AssetsPath = AssetsPath('raw-data/primary-sources/primary-sources.json'), MEX_LDAP_URL: SecretStr = SecretStr('**********'), MEX_LDAP_SEARCH_BASE: str = 'dc=ldapmock,dc=local', MEX_WIKI_API_URL: HttpUrl = HttpUrl('http://wikidata/'), MEX_WEB_USER_AGENT: str = 'rki/mex', MEX_ORCID_API_URL: HttpUrl = HttpUrl('https://orcid/'), MEX_BACKEND_HOST: Annotated[str, MinLen(min_length=1), MaxLen(max_length=250)] = 'localhost', MEX_BACKEND_PORT: Annotated[int, Gt(gt=0), Lt(lt=65536)] = 8080, MEX_BACKEND_ROOT_PATH: str = '', MEX_GRAPH_URL: str = 'neo4j://localhost:7687', MEX_GRAPH_NAME: str = 'neo4j', MEX_GRAPH_USER: SecretStr = SecretStr('**********'), MEX_GRAPH_PASSWORD: SecretStr = SecretStr('**********'), MEX_GRAPH_TX_TIMEOUT: int | float = 15.0, MEX_GRAPH_SESSION_TIMEOUT: int | float = 45.0, MEX_BACKEND_NON_MATCHABLE_TYPES: list[MergedType] = [MergedType.MERGED_CONSENT, MergedType.MERGED_PERSON], MEX_BACKEND_API_KEY_DATABASE: APIKeyDatabase = APIKeyDatabase(read=[], write=[]), MEX_BACKEND_VALKEY_URL: SecretStr | None = None)

Bases: BaseSettings

Settings definition for the backend server.

assert_valkey_is_configured_when_parallelized() → Self

Validate that valkey is configured if parallelization is > 1.

Rationale: We cache identities to make sure that multiple calls for getting an identifier receive the same identifier, even if the item with this identifier is not yet ingested in our database. If we use multiple backend instances, they must use a shared cache for storing these identities. The only shared cache is valkey, hence we make sure that valkey is configured if parallelization > 1.

backend_api_key_database: APIKeyDatabase

backend_host: str

backend_port: int

backend_root_path: str

graph_db: str

graph_password: SecretStr

graph_session_timeout: int | float

graph_tx_timeout: int | float

graph_url: str

graph_user: SecretStr

model_config = {'arbitrary_types_allowed': True, 'case_sensitive': False, 'cli_avoid_json': False, 'cli_enforce_required': False, 'cli_exit_on_error': True, 'cli_flag_prefix_char': '-', 'cli_hide_none_type': False, 'cli_ignore_unknown_args': False, 'cli_implicit_flags': False, 'cli_kebab_case': False, 'cli_parse_args': None, 'cli_parse_none_str': None, 'cli_prefix': '', 'cli_prog_name': None, 'cli_shortcuts': None, 'cli_use_class_docs_for_groups': False, 'enable_decoding': True, 'env_file': '.env', 'env_file_encoding': 'utf-8', 'env_ignore_empty': False, 'env_nested_delimiter': '__', 'env_nested_max_split': None, 'env_parse_enums': None, 'env_parse_none_str': None, 'env_prefix': 'mex_', 'env_prefix_target': 'variable', 'extra': 'ignore', 'json_file': None, 'json_file_encoding': None, 'nested_model_default_partial_update': False, 'populate_by_name': True, 'protected_namespaces': ('model_validate', 'model_dump', 'settings_customise_sources'), 'secrets_dir': None, 'toml_file': None, 'validate_assignment': True, 'validate_by_alias': True, 'validate_by_name': True, 'validate_default': True, 'yaml_config_section': None, 'yaml_file': None, 'yaml_file_encoding': None}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

non_matchable_types: list[MergedType]

valkey_url: SecretStr | None

mex.backend.types module

class mex.backend.types.APIKey(secret_value: SecretType)

Bases: SecretStr

An API Key used for authenticating and authorizing a client.

class mex.backend.types.AccessLevel(*values)

Bases: Enum

Enum of access level.

READ = 'read'

WRITE = 'write'

class mex.backend.types.DynamicStrEnum(name: str, bases: tuple[type], dct: EnumDict)

Bases: EnumType

Metaclass to dynamically populate an enumeration from a list of strings.

class mex.backend.types.ExtractedType(*values)

Bases: Enum

Enumeration of possible types for extracted items.

EXTRACTED_ACCESS_PLATFORM = 'ExtractedAccessPlatform'

EXTRACTED_ACTIVITY = 'ExtractedActivity'

EXTRACTED_BIBLIOGRAPHIC_RESOURCE = 'ExtractedBibliographicResource'

EXTRACTED_CONSENT = 'ExtractedConsent'

EXTRACTED_CONTACT_POINT = 'ExtractedContactPoint'

EXTRACTED_DISTRIBUTION = 'ExtractedDistribution'

EXTRACTED_ORGANIZATION = 'ExtractedOrganization'

EXTRACTED_ORGANIZATIONAL_UNIT = 'ExtractedOrganizationalUnit'

EXTRACTED_PERSON = 'ExtractedPerson'

EXTRACTED_PRIMARY_SOURCE = 'ExtractedPrimarySource'

EXTRACTED_RESOURCE = 'ExtractedResource'

EXTRACTED_VARIABLE = 'ExtractedVariable'

EXTRACTED_VARIABLE_GROUP = 'ExtractedVariableGroup'

class mex.backend.types.MergedType(*values)

Bases: Enum

Enumeration of possible types for merged items.

MERGED_ACCESS_PLATFORM = 'MergedAccessPlatform'

MERGED_ACTIVITY = 'MergedActivity'

MERGED_BIBLIOGRAPHIC_RESOURCE = 'MergedBibliographicResource'

MERGED_CONSENT = 'MergedConsent'

MERGED_CONTACT_POINT = 'MergedContactPoint'

MERGED_DISTRIBUTION = 'MergedDistribution'

MERGED_ORGANIZATION = 'MergedOrganization'

MERGED_ORGANIZATIONAL_UNIT = 'MergedOrganizationalUnit'

MERGED_PERSON = 'MergedPerson'

MERGED_PRIMARY_SOURCE = 'MergedPrimarySource'

MERGED_RESOURCE = 'MergedResource'

MERGED_VARIABLE = 'MergedVariable'

MERGED_VARIABLE_GROUP = 'MergedVariableGroup'

class mex.backend.types.ReferenceFieldName(*values)

Bases: Enum

Enumeration of possible field names that contain references.

ACCESS_PLATFORM = 'accessPlatform'

ACCESS_SERVICE = 'accessService'

AFFILIATION = 'affiliation'

ANALYTICS = 'analytics'

BELONGS_TO = 'belongsTo'

CONTACT = 'contact'

CONTAINED_BY = 'containedBy'

CONTRIBUTING_UNIT = 'contributingUnit'

CONTRIBUTOR = 'contributor'

CREATOR = 'creator'

DISTRIBUTION = 'distribution'

EDITOR = 'editor'

EDITOR_OF_SERIES = 'editorOfSeries'

EXTERNAL_ASSOCIATE = 'externalAssociate'

EXTERNAL_PARTNER = 'externalPartner'

FUNDER_OR_COMMISSIONER = 'funderOrCommissioner'

HAD_PRIMARY_SOURCE = 'hadPrimarySource'

HAS_DATA_SUBJECT = 'hasDataSubject'

INVOLVED_PERSON = 'involvedPerson'

INVOLVED_UNIT = 'involvedUnit'

IS_PART_OF = 'isPartOf'

IS_PART_OF_ACTIVITY = 'isPartOfActivity'

MEMBER_OF = 'memberOf'

PARENT_UNIT = 'parentUnit'

PUBLICATION = 'publication'

PUBLISHER = 'publisher'

RELATED_ACTIVITY = 'relatedActivity'

RELATED_RESOURCE = 'relatedResource'

RESPONSIBLE_UNIT = 'responsibleUnit'

SAMPLE = 'sample'

SOURCE = 'source'

STABLE_TARGET_ID = 'stableTargetId'

SUCCEEDS = 'succeeds'

SUPERSEDED_BY = 'supersededBy'

UNIT_IN_CHARGE = 'unitInCharge'

UNIT_OF = 'unitOf'

USED_IN = 'usedIn'

WAS_GENERATED_BY = 'wasGeneratedBy'

Module contents