Skip to content

Bases: ModelFactoryMixin

A Connector allows to connect and access data stored in various places. The list of available connectors can be found here.

Parameters:

Name Type Description Default
connector_type Union[ConnectorType, str]

Type of the connector to be created

 None
credentials dict

Connector credentials

 None
name Optional[str]

(optional) Connector name

 None
client Client

(optional) Client to connect to the backend

 None

Attributes:

Name Type Description
uid UID

UID fo the connector instance (creating internally)
type ConnectorType

Type of the connector
Source code in ydata/sdk/connectors/connector.py 
 19
 20
 21
 22
 23
 24
 25
 26
 27
 28
 29
 30
 31
 32
 33
 34
 35
 36
 37
 38
 39
 40
 41
 42
 43
 44
 45
 46
 47
 48
 49
 50
 51
 52
 53
 54
 55
 56
 57
 58
 59
 60
 61
 62
 63
 64
 65
 66
 67
 68
 69
 70
 71
 72
 73
 74
 75
 76
 77
 78
 79
 80
 81
 82
 83
 84
 85
 86
 87
 88
 89
 90
 91
 92
 93
 94
 95
 96
 97
 98
 99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
class Connector(ModelFactoryMixin):
    """A [`Connector`][ydata.sdk.connectors.Connector] allows to connect and
    access data stored in various places. The list of available connectors can
    be found [here][ydata.sdk.connectors.ConnectorType].

    Arguments:
        connector_type (Union[ConnectorType, str]): Type of the connector to be created
        credentials (dict): Connector credentials
        name (Optional[str]): (optional) Connector name
        client (Client): (optional) Client to connect to the backend

    Attributes:
        uid (UID): UID fo the connector instance (creating internally)
        type (ConnectorType): Type of the connector
    """

    def __init__(self, connector_type: Union[ConnectorType, str] = None, credentials: Optional[Dict] = None,  name: Optional[str] = None, client: Optional[Client] = None):
        self._init_common(client=client)
        self._model: Optional[mConnector] = self._create_model(
            connector_type, credentials, name, client=client)

    @init_client
    def _init_common(self, client: Optional[Client] = None):
        self._client = client
        self._logger = create_logger(__name__, level=LOG_LEVEL)

    @property
    def uid(self) -> UID:
        return self._model.uid

    @property
    def type(self) -> str:
        return self._model.type

    @staticmethod
    @init_client
    def get(uid: UID, client: Optional[Client] = None) -> "Connector":
        """Get an existing connector.

        Arguments:
            uid (UID): Connector identifier
            client (Client): (optional) Client to connect to the backend

        Returns:
            Connector
        """
        connectors: ConnectorsList = Connector.list(client=client)
        data = connectors.get_by_uid(uid)
        model = mConnector(**data)
        connector = ModelFactoryMixin._init_from_model_data(Connector, model)
        return connector

    @staticmethod
    def _init_connector_type(connector_type: Union[ConnectorType, str]) -> ConnectorType:
        if isinstance(connector_type, str):
            try:
                connector_type = ConnectorType(connector_type)
            except Exception:
                c_list = ", ".join([c.value for c in ConnectorType])
                raise InvalidConnectorError(
                    f"ConnectorType '{connector_type}' does not exist.\nValid connector types are: {c_list}.")
        return connector_type

    @staticmethod
    def _init_credentials(connector_type: ConnectorType, credentials: Union[str, Path, Dict, Credentials]) -> Credentials:
        _credentials = None

        if isinstance(credentials, str):
            credentials = Path(credentials)

        if isinstance(credentials, Path):
            try:
                _credentials = json_loads(credentials.open().read())
            except Exception:
                raise CredentialTypeError(
                    'Could not read the credentials. Please, check your path or credentials structure.')

        try:
            from ydata.sdk.connectors._models.connector_map import TYPE_TO_CLASS
            credential_cls = TYPE_TO_CLASS.get(connector_type.value)
            _credentials = credential_cls(**_credentials)
        except Exception:
            raise CredentialTypeError(
                "Could not create the credentials. Verify the path or the structure your credentials.")

        return _credentials

    @staticmethod
    def create(connector_type: Union[ConnectorType, str], credentials: Union[str, Path, Dict, Credentials], name: Optional[str] = None, client: Optional[Client] = None) -> "Connector":
        """Create a new connector.

        Arguments:
            connector_type (Union[ConnectorType, str]): Type of the connector to be created
            credentials (dict): Connector credentials
            name (Optional[str]): (optional) Connector name
            client (Client): (optional) Client to connect to the backend

        Returns:
            New connector
        """
        model = Connector._create_model(
            connector_type=connector_type, credentials=credentials, name=name, client=client)
        connector = ModelFactoryMixin._init_from_model_data(
            Connector, model)
        return connector

    @classmethod
    @init_client
    def _create_model(cls, connector_type: Union[ConnectorType, str], credentials: Union[str, Path, Dict, Credentials], name: Optional[str] = None, client: Optional[Client] = None) -> mConnector:
        _name = name if name is not None else str(uuid4())
        _connector_type = Connector._init_connector_type(connector_type)
        _credentials = Connector._init_credentials(_connector_type, credentials)
        payload = {
            "type": _connector_type.value,
            "credentials": _credentials.dict(by_alias=True),
            "name": _name
        }
        response = client.post('/connector/', json=payload)
        data: list = response.json()

        return mConnector(**data)

    @staticmethod
    @init_client
    def list(client: Optional[Client] = None) -> ConnectorsList:
        """List the connectors instances.

        Arguments:
            client (Client): (optional) Client to connect to the backend

        Returns:
            List of connectors
        """
        response = client.get('/connector')
        data: list = response.json()
        return ConnectorsList(data)

    def __repr__(self):
        return self._model.__repr__()

create(connector_type, credentials, name=None, client=None) staticmethod

Create a new connector.

Parameters:

Name Type Description Default
connector_type Union[ConnectorType, str]

Type of the connector to be created

 required
credentials dict

Connector credentials

 required
name Optional[str]

(optional) Connector name

 None
client Client

(optional) Client to connect to the backend

 None

Returns:

Type Description
Connector

New connector
Source code in ydata/sdk/connectors/connector.py 
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
@staticmethod
def create(connector_type: Union[ConnectorType, str], credentials: Union[str, Path, Dict, Credentials], name: Optional[str] = None, client: Optional[Client] = None) -> "Connector":
    """Create a new connector.

    Arguments:
        connector_type (Union[ConnectorType, str]): Type of the connector to be created
        credentials (dict): Connector credentials
        name (Optional[str]): (optional) Connector name
        client (Client): (optional) Client to connect to the backend

    Returns:
        New connector
    """
    model = Connector._create_model(
        connector_type=connector_type, credentials=credentials, name=name, client=client)
    connector = ModelFactoryMixin._init_from_model_data(
        Connector, model)
    return connector

get(uid, client=None) staticmethod

Get an existing connector.

Parameters:

Name Type Description Default
uid UID

Connector identifier

 required
client Client

(optional) Client to connect to the backend

 None

Returns:

Type Description
Connector

Connector
Source code in ydata/sdk/connectors/connector.py 
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
@staticmethod
@init_client
def get(uid: UID, client: Optional[Client] = None) -> "Connector":
    """Get an existing connector.

    Arguments:
        uid (UID): Connector identifier
        client (Client): (optional) Client to connect to the backend

    Returns:
        Connector
    """
    connectors: ConnectorsList = Connector.list(client=client)
    data = connectors.get_by_uid(uid)
    model = mConnector(**data)
    connector = ModelFactoryMixin._init_from_model_data(Connector, model)
    return connector

list(client=None) staticmethod

List the connectors instances.

Parameters:

Name Type Description Default
client Client

(optional) Client to connect to the backend

 None

Returns:

Type Description
ConnectorsList

List of connectors
Source code in ydata/sdk/connectors/connector.py 
141
142
143
144
145
146
147
148
149
150
151
152
153
154
@staticmethod
@init_client
def list(client: Optional[Client] = None) -> ConnectorsList:
    """List the connectors instances.

    Arguments:
        client (Client): (optional) Client to connect to the backend

    Returns:
        List of connectors
    """
    response = client.get('/connector')
    data: list = response.json()
    return ConnectorsList(data)

ConnectorType

Bases: Enum

AWS_S3 = 'aws-s3' class-attribute

AWS S3 connector

AZURE_BLOB = 'azure-blob' class-attribute

Azure Blob connector

AZURE_SQL = 'azure-sql' class-attribute

AzureSQL connector

BIGQUERY = 'google-bigquery' class-attribute

BigQuery connector

FILE = 'file' class-attribute

File connector (placeholder)

GCS = 'gcs' class-attribute

Google Cloud Storage connector

MYSQL = 'mysql' class-attribute

MySQL connector

SNOWFLAKE = 'snowflake' class-attribute

Snowflake connector