mex.backend package

Subpackages

• mex.backend.auxiliary package
  • Submodules
  • mex.backend.auxiliary.ldap module
    • search_persons_or_contact_points_in_ldap()
  • mex.backend.auxiliary.orcid module
    • search_persons_in_orcid()
  • mex.backend.auxiliary.organigram module
    • extracted_organizational_units()
  • mex.backend.auxiliary.primary_source module
    • _fetch_or_insert_primary_source()
    • extracted_primary_source_ldap()
    • extracted_primary_source_orcid()
    • extracted_primary_source_organigram()
    • extracted_primary_source_wikidata()
  • mex.backend.auxiliary.utils module
    • fetch_extracted_item_by_source_identifiers()
  • mex.backend.auxiliary.wikidata module
    • _fetch_or_insert_organization()
    • extracted_organization_rki()
    • search_organizations_in_wikidata()
  • Module contents
• mex.backend.cache package
  • Submodules
  • mex.backend.cache.connector module
    • CacheConnector
      • CacheConnector.__init__()
      • CacheConnector.close()
      • CacheConnector.delete_value()
      • CacheConnector.flush()
      • CacheConnector.get_value()
      • CacheConnector.metrics()
      • CacheConnector.set_value()
    • LocalCache
      • LocalCache.close()
      • LocalCache.delete()
      • LocalCache.flushdb()
      • LocalCache.info()
      • LocalCache.set()
    • ValkeyCache
      • ValkeyCache.__init__()
      • ValkeyCache.close()
      • ValkeyCache.delete()
      • ValkeyCache.flushdb()
      • ValkeyCache.get()
      • ValkeyCache.info()
      • ValkeyCache.set()
  • Module contents
• mex.backend.extracted package
  • Submodules
  • mex.backend.extracted.helpers module
    • get_extracted_item_from_graph()
    • get_extracted_items_from_graph()
    • search_extracted_items_in_graph()
  • mex.backend.extracted.main module
    • get_extracted_item()
    • search_extracted_items()
  • Module contents
• mex.backend.graph package
  • Submodules
  • mex.backend.graph.connector module
    • GraphConnector
      • GraphConnector.__init__()
      • GraphConnector._check_connectivity_and_authentication()
      • GraphConnector._check_match_preconditions_tx()
      • GraphConnector._fetch_extracted_or_rule_items()
      • GraphConnector._init_driver()
      • GraphConnector._match_item_tx()
      • GraphConnector._run_ingest_in_transaction()
      • GraphConnector._seed_constraints()
      • GraphConnector._seed_data()
      • GraphConnector._seed_indices()
      • GraphConnector.close()
      • GraphConnector.commit()
      • GraphConnector.delete_item()
      • GraphConnector.delete_rule_set()
      • GraphConnector.exists_item()
      • GraphConnector.fetch_extracted_items()
      • GraphConnector.fetch_identities()
      • GraphConnector.fetch_merged_items()
      • GraphConnector.fetch_rule_items()
      • GraphConnector.flush()
      • GraphConnector.ingest_items()
      • GraphConnector.match_item()
  • mex.backend.graph.exceptions module
    • DeletionFailedError
    • GraphError
      • GraphError.__init__()
      • GraphError.errors()
      • GraphError.is_retryable()
    • InconsistentGraphError
    • IngestionError
    • MatchingError
    • MultipleResultsFoundError
    • NoResultFoundError
  • mex.backend.graph.models module
    • EdgeExporter
      • EdgeExporter.transform()
    • ExtractedPrimarySourceWithHardcodedIdentifiers
      • ExtractedPrimarySourceWithHardcodedIdentifiers.entityType
      • ExtractedPrimarySourceWithHardcodedIdentifiers.identifier
      • ExtractedPrimarySourceWithHardcodedIdentifiers.model_config
      • ExtractedPrimarySourceWithHardcodedIdentifiers.stableTargetId
    • GraphRel
      • GraphRel.edgeLabel
      • GraphRel.edgeProps
      • GraphRel.nodeLabels
      • GraphRel.nodeProps
    • IngestData
      • IngestData.createRels
      • IngestData.entityType
      • IngestData.identifier
      • IngestData.linkRels
      • IngestData.metadata()
      • IngestData.model_config
      • IngestData.nodeProps
      • IngestData.sort_create_rels()
      • IngestData.sort_link_rels()
      • IngestData.stableTargetId
    • IngestParams
      • IngestParams.delete_node_edges
      • IngestParams.detach_node_edges
      • IngestParams.has_create_rels
      • IngestParams.has_link_rels
      • IngestParams.merged_label
      • IngestParams.model_config
      • IngestParams.node_label
      • IngestParams.used_nested_labels
      • IngestParams.used_referenced_labels
    • Result
      • Result.__init__()
      • Result.all()
      • Result.get_update_counters()
      • Result.log_notifications()
      • Result.one()
      • Result.one_or_none()
  • mex.backend.graph.query module
    • Query
      • Query.__init__()
      • Query.render()
    • QueryBuilder
      • QueryBuilder.__init__()
      • QueryBuilder._get_ingest_query_for_entity_type()
      • QueryBuilder.close()
    • render_constraints()
  • mex.backend.graph.transform module
    • _SearchResultReference
      • _SearchResultReference.label
      • _SearchResultReference.position
      • _SearchResultReference.value
    • expand_references_in_search_result()
    • get_error_details_from_neo4j_error()
    • get_graph_rel_id()
    • transform_model_into_ingest_data()
    • validate_ingested_data()
  • Module contents
• mex.backend.identity package
  • Submodules
  • mex.backend.identity.main module
    • assign_identity()
    • fetch_identity()
  • mex.backend.identity.provider module
    • GraphIdentityProvider
      • GraphIdentityProvider.__init__()
      • GraphIdentityProvider.assign()
      • GraphIdentityProvider.close()
      • GraphIdentityProvider.fetch()
      • GraphIdentityProvider.metrics()
  • Module contents
• mex.backend.ingest package
  • Submodules
  • mex.backend.ingest.main module
    • ingest_items()
  • Module contents
• mex.backend.ldap package
  • Submodules
  • mex.backend.ldap.main module
    • get_merged_person_from_login()
  • Module contents
• mex.backend.match package
  • Submodules
  • mex.backend.match.helpers module
    • match_item_in_graph()
  • mex.backend.match.main module
    • match_item()
  • Module contents
• mex.backend.merged package
  • Submodules
  • mex.backend.merged.helpers module
    • delete_merged_item_from_graph()
    • get_merged_item_from_graph()
    • merge_search_result_item()
    • search_merged_items_in_graph()
  • mex.backend.merged.main module
    • delete_merged_item()
    • get_merged_item()
    • search_merged_items()
  • Module contents
• mex.backend.preview package
  • Submodules
  • mex.backend.preview.main module
    • preview_item()
    • search_preview_items()
  • Module contents
• mex.backend.rules package
  • Submodules
  • mex.backend.rules.helpers module
    • create_and_get_rule_set()
    • delete_rule_set_from_graph()
    • get_rule_set_from_graph()
    • update_and_get_rule_set()
  • mex.backend.rules.main module
    • create_rule_set()
    • delete_rule_set()
    • get_rule_set()
    • update_rule_set()
  • mex.backend.rules.transform module
    • transform_raw_rules_to_rule_set_response()
  • Module contents
• mex.backend.system package
  • Submodules
  • mex.backend.system.main module
    • check_system_status()
    • get_prometheus_metrics()
  • mex.backend.system.models module
    • GraphStatus
      • GraphStatus.model_config
      • GraphStatus.status
    • SystemStatus
      • SystemStatus.model_config
      • SystemStatus.status
      • SystemStatus.version
  • Module contents
• mex.backend.testing package
  • Submodules
  • mex.backend.testing.ldap module
    • get_merged_person_from_login()
    • search_persons_or_contact_points_in_ldap()
  • mex.backend.testing.main module
  • mex.backend.testing.security module
    • has_write_access_ldap_mocked()
  • mex.backend.testing.system module
    • flush_graph_database()
  • Module contents

Submodules

mex.backend.exceptions module

exception mex.backend.exceptions.BackendError

Bases: MExError

Base backend error that offer details on underlying pydantic errors.

errors() → list[ErrorDetails]

Details about underlying pydantic errors.

is_retryable() → bool

Whether the error is retryable.

class mex.backend.exceptions.DebuggingInfo(*, errors: list[dict[str, Any]], scope: DebuggingScope)

Bases: BaseModel

Debugging information for error responses.

errors: list[dict[str, Any]]

model_config = {}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

scope: DebuggingScope

class mex.backend.exceptions.DebuggingScope(*, http_version: str, method: str, path: str, path_params: dict[str, Any], query_string: str, scheme: str)

Bases: BaseModel

Scope for debugging info of error responses.

http_version: str

method: str

model_config = {'extra': 'ignore'}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

path: str

path_params: dict[str, Any]

query_string: str

scheme: str

class mex.backend.exceptions.DetailedError(*args, **kwargs)

Bases: Protocol

Protocol for errors that offer details.

errors() → list[ErrorDetails]

Details about each underlying error.

class mex.backend.exceptions.ErrorResponse(*, message: str, debug: DebuggingInfo)

Bases: BaseModel

Response model for user and system errors.

debug: DebuggingInfo

message: str

model_config = {}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

mex.backend.exceptions.handle_detailed_error(request: Request, exc: Exception) → Response

Handle detailed errors and provide debugging info.

mex.backend.exceptions.handle_uncaught_exception(request: Request, exc: Exception) → Response

Handle uncaught errors and provide debugging info.

mex.backend.logging module

mex.backend.main module

mex.backend.main.lifespan(_: FastAPI) → AsyncIterator[None]

Async context manager to execute startup and teardown of the FastAPI app.

mex.backend.models module

class mex.backend.models.APIKeyDatabase(*, read: list[APIKey] = [], write: list[APIKey] = [])

Bases: BaseModel

A lookup from access level to list of allowed APIKeys.

model_config = {}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

read: list[APIKey]

write: list[APIKey]

class mex.backend.models.APIUserDatabase(*, read: dict[str, APIUserPassword] = {}, write: dict[str, APIUserPassword] = {})

Bases: BaseModel

Database containing usernames and passwords for backend API.

model_config = {}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

read: dict[str, APIUserPassword]

write: dict[str, APIUserPassword]

mex.backend.security module

mex.backend.security.check_header_for_authorization_method(api_key: Annotated[str | None, ~fastapi.params.Depends(dependency=<fastapi.security.api_key.APIKeyHeader object at 0x7f225d8ceba0>, use_cache=True, scope=None)] = None, credentials: HTTPBasicCredentials | None, ~fastapi.params.Depends(dependency=<fastapi.security.http.HTTPBasic object at 0x7f225d8cecf0>, use_cache=True, scope=None)] = None, user_agent: Annotated[str, Header(PydanticUndefined)]='n/a') → None

Check authorization header for API key or credentials.

Raises:

HTTPException if both API key and credentials or none of them are in header. –

Parameters:

• api_key – the API key

• credentials – username and password

• user_agent – user-agent (in case of a web browser starts with “Mozilla/”)

mex.backend.security.has_read_access(api_key: Annotated[str | None, ~fastapi.params.Depends(dependency=<fastapi.security.api_key.APIKeyHeader object at 0x7f225d8ceba0>, use_cache=True, scope=None)] = None, credentials: HTTPBasicCredentials | None, ~fastapi.params.Depends(dependency=<fastapi.security.http.HTTPBasic object at 0x7f225d8cecf0>, use_cache=True, scope=None)] = None, user_agent: Annotated[str, Header(PydanticUndefined)]='n/a') → None

Verify if api key or credentials have read access or write access.

Raises:

HTTPException if no header or provided APIKey/credentials have no read access. –

Parameters:

• api_key – the API key

• credentials – username and password

• user_agent – user-agent (in case of a web browser starts with “Mozilla/”)

Settings:

check credentials in backend_user_database or backend_api_key_database

mex.backend.security.has_write_access(api_key: Annotated[str | None, ~fastapi.params.Depends(dependency=<fastapi.security.api_key.APIKeyHeader object at 0x7f225d8ceba0>, use_cache=True, scope=None)] = None, credentials: HTTPBasicCredentials | None, ~fastapi.params.Depends(dependency=<fastapi.security.http.HTTPBasic object at 0x7f225d8cecf0>, use_cache=True, scope=None)] = None, user_agent: Annotated[str, Header(PydanticUndefined)]='n/a') → None

Verify if provided api key or credentials have write access.

Raises:

HTTPException if no header or provided APIKey/credentials have no write access. –

Parameters:

• api_key – the API key

• credentials – username and password

• user_agent – user-agent (in case of a web browser starts with “Mozilla/”)

Settings:

check credentials in backend_user_database or backend_api_key_database

mex.backend.security.has_write_access_ldap(credentials: ~typing.Annotated[~fastapi.security.http.HTTPBasicCredentials, ~fastapi.params.Depends(dependency=<fastapi.security.http.HTTPBasic object at 0x7f225d8cecf0>, use_cache=True, scope=None)]) → str

Verify if provided credentials have LDAP write access.

Raises:

HTTPException if credentials have no LDAP write access or are missing. –

Parameters:

credentials – username and password

mex.backend.settings module

class mex.backend.settings.BackendSettings(_env_file: Path | str | Sequence[Path | str] | None = PosixPath('.'), _env_file_encoding: str | None = None, _env_nested_delimiter: str | None = None, _secrets_dir: str | Path | None = None, *, MEX_DEBUG: bool = False, MEX_SINK: list[Sink] = [Sink.NDJSON], MEX_ASSETS_DIR: Path = PosixPath('/home/runner/work/mex-backend/mex-backend/assets'), MEX_WORK_DIR: Path = PosixPath('/home/runner/work/mex-backend/mex-backend'), MEX_IDENTITY_PROVIDER: IdentityProvider = IdentityProvider.MEMORY, MEX_BACKEND_API_URL: HttpUrl = HttpUrl('http://localhost:8080/'), MEX_BACKEND_API_KEY: SecretStr = SecretStr('**********'), MEX_BACKEND_API_PARALLELIZATION: int = 1, MEX_BACKEND_API_CHUNK_SIZE: int = 25, MEX_VERIFY_SESSION: bool | AssetsPath = True, MEX_ORGANIGRAM_PATH: AssetsPath = AssetsPath('raw-data/organigram/organizational_units.json'), MEX_PRIMARY_SOURCES_PATH: AssetsPath = AssetsPath('raw-data/primary-sources/primary-sources.json'), MEX_LDAP_URL: SecretStr = SecretStr('**********'), MEX_LDAP_SEARCH_BASE: str = 'dc=ldapmock,dc=local', MEX_WIKI_API_URL: HttpUrl = HttpUrl('http://wikidata/'), MEX_WEB_USER_AGENT: str = 'rki/mex', MEX_ORCID_API_URL: HttpUrl = HttpUrl('https://orcid/'), MEX_BACKEND_HOST: Annotated[str, MinLen(min_length=1), MaxLen(max_length=250)] = 'localhost', MEX_BACKEND_PORT: Annotated[int, Gt(gt=0), Lt(lt=65536)] = 8080, MEX_BACKEND_ROOT_PATH: str = '', MEX_GRAPH_URL: str = 'neo4j://localhost:7687', MEX_GRAPH_NAME: str = 'neo4j', MEX_GRAPH_USER: SecretStr = SecretStr('**********'), MEX_GRAPH_PASSWORD: SecretStr = SecretStr('**********'), MEX_GRAPH_TX_TIMEOUT: int | float = 15.0, MEX_GRAPH_SESSION_TIMEOUT: int | float = 45.0, MEX_BACKEND_NON_MATCHABLE_TYPES: list[MergedType] = [MergedType.MERGED_CONSENT, MergedType.MERGED_PERSON], MEX_BACKEND_API_KEY_DATABASE: APIKeyDatabase = APIKeyDatabase(read=[], write=[]), MEX_BACKEND_API_USER_DATABASE: APIUserDatabase = APIUserDatabase(read={}, write={}), MEX_BACKEND_VALKEY_URL: SecretStr | None = None)

Bases: BaseSettings

Settings definition for the backend server.

assert_valkey_is_configured_when_parallelized() → Self

Validate that valkey is configured if parallelization is > 1.

Rationale: We cache identities to make sure that multiple calls for getting an identifier receive the same identifier, even if the item with this identifier is not yet ingested in our database. If we use multiple backend instances, they must use a shared cache for storing these identities. The only shared cache is valkey, hence we make sure that valkey is configured if parallelization > 1.

backend_api_key_database: APIKeyDatabase

backend_host: str

backend_port: int

backend_root_path: str

backend_user_database: APIUserDatabase

graph_db: str

graph_password: SecretStr

graph_session_timeout: int | float

graph_tx_timeout: int | float

graph_url: str

graph_user: SecretStr

model_config = {'arbitrary_types_allowed': True, 'case_sensitive': False, 'cli_avoid_json': False, 'cli_enforce_required': False, 'cli_exit_on_error': True, 'cli_flag_prefix_char': '-', 'cli_hide_none_type': False, 'cli_ignore_unknown_args': False, 'cli_implicit_flags': False, 'cli_kebab_case': False, 'cli_parse_args': None, 'cli_parse_none_str': None, 'cli_prefix': '', 'cli_prog_name': None, 'cli_shortcuts': None, 'cli_use_class_docs_for_groups': False, 'enable_decoding': True, 'env_file': '.env', 'env_file_encoding': 'utf-8', 'env_ignore_empty': False, 'env_nested_delimiter': '__', 'env_nested_max_split': None, 'env_parse_enums': None, 'env_parse_none_str': None, 'env_prefix': 'mex_', 'env_prefix_target': 'variable', 'extra': 'ignore', 'json_file': None, 'json_file_encoding': None, 'nested_model_default_partial_update': False, 'populate_by_name': True, 'protected_namespaces': ('model_validate', 'model_dump', 'settings_customise_sources'), 'secrets_dir': None, 'toml_file': None, 'validate_assignment': True, 'validate_by_alias': True, 'validate_by_name': True, 'validate_default': True, 'yaml_config_section': None, 'yaml_file': None, 'yaml_file_encoding': None}

Configuration for the model, should be a dictionary conforming to [ConfigDict][pydantic.config.ConfigDict].

non_matchable_types: list[MergedType]

valkey_url: SecretStr | None

mex.backend.types module

class mex.backend.types.APIKey(secret_value: SecretType)

Bases: SecretStr

An API Key used for authenticating and authorizing a client.

class mex.backend.types.APIUserPassword(secret_value: SecretType)

Bases: SecretStr

An API password used for basic authentication along with a username.

class mex.backend.types.AccessLevel(*values)

Bases: Enum

Enum of access level.

READ = 'read'

WRITE = 'write'

class mex.backend.types.DynamicStrEnum(name: str, bases: tuple[type], dct: EnumDict)

Bases: EnumType

Metaclass to dynamically populate an enumeration from a list of strings.

class mex.backend.types.ExtractedType(*values)

Bases: Enum

Enumeration of possible types for extracted items.

EXTRACTED_ACCESS_PLATFORM = 'ExtractedAccessPlatform'

EXTRACTED_ACTIVITY = 'ExtractedActivity'

EXTRACTED_BIBLIOGRAPHIC_RESOURCE = 'ExtractedBibliographicResource'

EXTRACTED_CONSENT = 'ExtractedConsent'

EXTRACTED_CONTACT_POINT = 'ExtractedContactPoint'

EXTRACTED_DISTRIBUTION = 'ExtractedDistribution'

EXTRACTED_ORGANIZATION = 'ExtractedOrganization'

EXTRACTED_ORGANIZATIONAL_UNIT = 'ExtractedOrganizationalUnit'

EXTRACTED_PERSON = 'ExtractedPerson'

EXTRACTED_PRIMARY_SOURCE = 'ExtractedPrimarySource'

EXTRACTED_RESOURCE = 'ExtractedResource'

EXTRACTED_VARIABLE = 'ExtractedVariable'

EXTRACTED_VARIABLE_GROUP = 'ExtractedVariableGroup'

class mex.backend.types.MergedType(*values)

Bases: Enum

Enumeration of possible types for merged items.

MERGED_ACCESS_PLATFORM = 'MergedAccessPlatform'

MERGED_ACTIVITY = 'MergedActivity'

MERGED_BIBLIOGRAPHIC_RESOURCE = 'MergedBibliographicResource'

MERGED_CONSENT = 'MergedConsent'

MERGED_CONTACT_POINT = 'MergedContactPoint'

MERGED_DISTRIBUTION = 'MergedDistribution'

MERGED_ORGANIZATION = 'MergedOrganization'

MERGED_ORGANIZATIONAL_UNIT = 'MergedOrganizationalUnit'

MERGED_PERSON = 'MergedPerson'

MERGED_PRIMARY_SOURCE = 'MergedPrimarySource'

MERGED_RESOURCE = 'MergedResource'

MERGED_VARIABLE = 'MergedVariable'

MERGED_VARIABLE_GROUP = 'MergedVariableGroup'

class mex.backend.types.ReferenceFieldName(*values)

Bases: Enum

Enumeration of possible field names that contain references.

ACCESS_PLATFORM = 'accessPlatform'

ACCESS_SERVICE = 'accessService'

AFFILIATION = 'affiliation'

BELONGS_TO = 'belongsTo'

CONTACT = 'contact'

CONTAINED_BY = 'containedBy'

CONTRIBUTING_UNIT = 'contributingUnit'

CONTRIBUTOR = 'contributor'

CREATOR = 'creator'

DISTRIBUTION = 'distribution'

EDITOR = 'editor'

EDITOR_OF_SERIES = 'editorOfSeries'

EXTERNAL_ASSOCIATE = 'externalAssociate'

EXTERNAL_PARTNER = 'externalPartner'

FUNDER_OR_COMMISSIONER = 'funderOrCommissioner'

HAD_PRIMARY_SOURCE = 'hadPrimarySource'

HAS_DATA_SUBJECT = 'hasDataSubject'

INVOLVED_PERSON = 'involvedPerson'

INVOLVED_UNIT = 'involvedUnit'

IS_PART_OF = 'isPartOf'

IS_PART_OF_ACTIVITY = 'isPartOfActivity'

MEMBER_OF = 'memberOf'

PARENT_UNIT = 'parentUnit'

PUBLICATION = 'publication'

PUBLISHER = 'publisher'

RESPONSIBLE_UNIT = 'responsibleUnit'

STABLE_TARGET_ID = 'stableTargetId'

SUCCEEDS = 'succeeds'

SUPERSEDED_BY = 'supersededBy'

UNIT_IN_CHARGE = 'unitInCharge'

UNIT_OF = 'unitOf'

USED_IN = 'usedIn'

WAS_GENERATED_BY = 'wasGeneratedBy'

Module contents