Filesystem

Create a list of directories from a config file

"""Example script for creating directories in Bash"""
import tamr_toolbox as tbox

tbox.filesystem.bash.create_directories(["/path/to/a/dir", "/path/to/different/dir"])

Clean old files recursively from a directory

"""Example script for cleaning up old files"""
from tamr_toolbox import filesystem


# to recursively delete files with a modification date older than the specified one
filesystem.bash.delete_old_files("/path/to/my/directory", num_days_to_keep=60)

# to exclude some paths underneath the top level directory
filesystem.bash.delete_old_files(
    "/path/to/my/directory", num_days_to_keep=60, exclude_paths=["save", "data/results"]
)

Cloud

Upload object to GCS bucket

"""
An example script to demonstrate the use of the gcs_upload function to upload file.
"""

import tamr_toolbox as tbox
from google.cloud import storage

gcs_client = storage.Client()

# uploads file to gcs
# upload a local file "my_local_directory/my_file.txt" to "gs://my-bucket/path-to-file"
tbox.filesystem.cloud.gcs_upload(
    cloud_client=gcs_client,
    source_filepath="my_local_directory/my_file.txt",
    destination_filepath="path-to-file",
    bucket_name="my-bucket",
)

Download object from GCS bucket

"""
An example script to demonstrate the use of the gcs_download function to download file,
from a GCS bucket.
"""

import tamr_toolbox as tbox
from google.cloud import storage

gcs_client = storage.Client()

# download file from gcs
# download a file on GCS "gs://my-bucket/path-to-file" to "my_local_directory/my_file.txt"
tbox.filesystem.cloud.gcs_download(
    cloud_client=gcs_client,
    source_filepath="path-to-file",
    destination_filepath="my_local_directory/my_file.txt",
    bucket_name="my-bucket",
)

Upload object to AWS S3 bucket

"""
An example script to demonstrate the use of the s3 function to upload file
"""

import tamr_toolbox as tbox
import boto3

s3_client = boto3.client("s3")

# uploads file to AWS S3
# upload a local file "my_local_directory/my_file.txt" to "s3://my-bucket/path-to-file"
tbox.filesystem.cloud.s3_upload(
    cloud_client=s3_client,
    source_filepath="my_local_directory/my_file.txt",
    destination_filepath="path-to-file",
    bucket_name="my-bucket",
)

Download object from AWS S3 bucket

"""
An example script to demonstrate the use of the s3_download function to download file,
from an S3 bucket.
"""

import tamr_toolbox as tbox
import boto3

s3_client = boto3.client("s3")

# download file from AWS S3
# download a file on GCS "s3://my-bucket/path-to-file" to "my_local_directory/my_file.txt"
tbox.filesystem.cloud.s3_download(
    cloud_client=s3_client,
    source_filepath="path-to-file",
    destination_filepath="my_local_directory/my_file.txt",
    bucket_name="my-bucket",
)