Automatically Infer Data Schema for Inserted Data

[jieguangzhou]

When inserting data, if the data is of a special type, the schema will be inferred automatically.

Steps:

1. Infer the data to get the schema.
2. Check if the table and schema are already created; if not, automatically create the table.
3. Insert the data.

Therefore, users no longer need to worry about the schema concept, unless they want to customize a new schema.

Here is the current user experience:

@pytest.mark.parametrize(
    "db", [DBConfig.mongodb_empty, DBConfig.sqldb_empty], indirect=True
)
def test_insert_with_schema(db):
    import numpy as np
    import PIL.Image

    from superduperdb.ext.numpy.encoder import NumpyDataTypeFactory
    from superduperdb.ext.pillow.encoder import pil_image

    data = {
        'img': PIL.Image.open('test/material/data/test.png'),
        'array': np.array([1, 2, 3]),
    }

    schema = Schema(
        'schema',
        fields={
            'img': pil_image,
            'array': NumpyDataTypeFactory.create(data['array']),
        },
    )

    table = Table('documents', schema=schema)
    db.add(table)

    table_or_collection = db['documents']
    datas = [Document(data)]

    table_or_collection.insert(datas).execute()

    datas_from_db = list(table_or_collection.select().execute())
    for d, d_db in zip(datas, datas_from_db):
        assert d['img'].size == d_db['img'].size
        assert np.all(d['array'] == d_db['array'])

Here is the optimized user experience:

@pytest.mark.parametrize(
    "db", [DBConfig.mongodb_empty, DBConfig.sqldb_empty], indirect=True
)
def test_insert_with_schema(db):
    import numpy as np
    import PIL.Image

    data = {
        'img': PIL.Image.open('test/material/data/test.png'),
        'array': np.array([1, 2, 3]),
    }

    table_or_collection = db['documents']
    datas = [Document(data)]

    table_or_collection.insert(datas).execute()

    datas_from_db = list(table_or_collection.select().execute())
    for d, d_db in zip(datas, datas_from_db):
        assert d['img'].size == d_db['img'].size
        assert np.all(d['array'] == d_db['array'])

[jieguangzhou]

close the same issue: #2068