Tests to see if stuff works and to illustrate a few points

create_test_folder[source]

create_test_folder(test_path:Path)

Create a folder in a location with dummy data ready to be registered

test_path = Path('test_env')
create_test_folder(test_path)

[!] test_env/01_Delivery/2020_01_13_no structure not processed, empty or non-existent
Writing receipt to test_env/01_Delivery/2020_01_10_iris_csv
Writing receipt to test_env/01_Delivery/2020_01_11_dump
Writing receipt to test_env/01_Delivery/2021_01_19_Iris
Writing receipt to test_env/01_Delivery/2020_01_09_near_earth_json
Writing receipt to test_env/01_Delivery/2020_01_12_missing_source
Writing receipt to test_env/01_Delivery/2020_01_08_video_files
Writing receipt to test_env/01_Delivery/2021_01_19_Mnist
Writing receipt to test_env/01_Delivery/2020_01_10_measurements_excel
Writing receipt to test_env/02_RAW/2020_02_03_iris_versicolor
Writing receipt to test_env/02_RAW/2020_02_02_iris_versicolor
Writing receipt to test_env/02_RAW/2020_02_03_missing_source
Writing receipt to test_env/02_RAW/2020_02_01_rick_roll_videos
Writing receipt to test_env/02_RAW/2020_02_02_metal_measurements
Writing receipt to test_env/02_RAW/2020_02_01_near_earth_objects
Writing receipt to test_env/02_RAW/2020_02_03_netflix_titles
Writing receipt to test_env/02_RAW/2020_02_02_temperature_mass_measurements
Writing receipt to test_env/03_Datasets/2020_02_04_iris_versicolor
Writing receipt to test_env/03_Datasets/2020_02_04_movie_titles
Writing receipt to test_env/03_Datasets/2020_02_04_asteroid_metal_mass

text = """

-------------
:Files Last Changed: 2021-02-10 20:20:18.899966
:File Size: 1244762
:Number of Files: 3
:Hash: 975bd61edb1adec7b61b8fb459de04e775d286bcc5cd19a4be67c74c04044e46

Report
------
None

"""
re.sub('(.*\nReport\n-+\n)None(\n.*)', '\\1Dummy HTML Report\\2', text, re.MULTILINE)

'\n\n-------------\n:Files Last Changed: 2021-02-10 20:20:18.899966\n:File Size: 1244762\n:Number of Files: 3\n:Hash: 975bd61edb1adec7b61b8fb459de04e775d286bcc5cd19a4be67c74c04044e46\n\nReport\n------\nDummy HTML Report\n\n'
env = DataIntakeEnv(test_path / '.config' / 'config.ini')

[sync_data_folder(env,group_type ) for group_type in [dm.Delivery, dm.Raw_Data, dm.Dataset] ]

Writing receipt to test_env/01_Delivery/2020_01_10_iris_csv
Writing receipt to test_env/01_Delivery/2020_01_11_dump
Writing receipt to test_env/01_Delivery/2021_01_19_Iris
Writing receipt to test_env/01_Delivery/2020_01_09_near_earth_json
Writing receipt to test_env/01_Delivery/2020_01_12_missing_source
Writing receipt to test_env/01_Delivery/2020_01_08_video_files
Writing receipt to test_env/01_Delivery/2021_01_19_Mnist
Writing receipt to test_env/01_Delivery/2020_01_10_measurements_excel
Writing receipt to test_env/02_RAW/2020_02_03_iris_versicolor
Writing receipt to test_env/02_RAW/2020_02_02_iris_versicolor
Writing receipt to test_env/02_RAW/2020_02_03_missing_source
Writing receipt to test_env/02_RAW/2020_02_01_rick_roll_videos
Writing receipt to test_env/02_RAW/2020_02_02_metal_measurements
Writing receipt to test_env/02_RAW/2020_02_01_near_earth_objects
Writing receipt to test_env/02_RAW/2020_02_03_netflix_titles
Writing receipt to test_env/02_RAW/2020_02_02_temperature_mass_measurements
Writing receipt to test_env/03_Datasets/2020_02_04_iris_versicolor
Writing receipt to test_env/03_Datasets/2020_02_04_movie_titles
Writing receipt to test_env/03_Datasets/2020_02_04_asteroid_metal_mass

[None, None, None]
[check_datagroup(env, group_type, light=False) for group_type in [dm.Delivery, dm.Raw_Data, dm.Dataset]]

[DLV] 2021_01_19_Iris no data extracted
[DLV] Registered data missing files or empty folder at test_env/01_Delivery/2020_01_13_no structure
[DLV] No Data registered for: 2020_01_13_no structure
[DLV] 2021_01_19_Mnist no data extracted



[['[DLV] 2021_01_19_Iris no data extracted',
  '[DLV] Registered data missing files or empty folder at test_env/01_Delivery/2020_01_13_no structure',
  '[DLV] No Data registered for: 2020_01_13_no structure',
  '[DLV] 2021_01_19_Mnist no data extracted'],
 [],
 []]
shutil.rmtree(test_path)