big_query_utils Namespace Reference

Functions
def	create_big_query ()
def	create_dataset (biq_query, project_id, dataset_id)
def	create_partitioned_table (big_query, project_id, dataset_id, table_id, table_schema, description, partition_type='DAY', expiration_ms=_EXPIRATION_MS)
def	create_table (big_query, project_id, dataset_id, table_id, table_schema, description)
def	create_table2 (big_query, project_id, dataset_id, table_id, fields_schema, description, partition_type=None, expiration_ms=None)
def	insert_rows (big_query, project_id, dataset_id, table_id, rows_list)
def	make_row (unique_row_id, row_values_dict)
def	patch_table (big_query, project_id, dataset_id, table_id, fields_schema)
def	sync_query_job (big_query, project_id, query, timeout=5000)

Variables
int	_EXPIRATION_MS = 30 * 24 * 60 * 60 * 1000
int	NUM_RETRIES = 3

Function Documentation

create_big_query()

def big_query_utils.create_big_query

(

)

Authenticates with cloud platform and gets a BiqQuery service object

Definition at line 31 of file tools/gcp/utils/big_query_utils.py.

create_dataset()

def big_query_utils.create_dataset	(		biq_query,
			project_id,
			dataset_id
	)

Definition at line 41 of file tools/gcp/utils/big_query_utils.py.

create_partitioned_table()

def big_query_utils.create_partitioned_table	(		big_query,
			project_id,
			dataset_id,
			table_id,
			table_schema,
			description,
			partition_type = 'DAY',
			expiration_ms = _EXPIRATION_MS
	)

Creates a partitioned table. By default, a date-paritioned table is created with
  each partition lasting 30 days after it was last modified.

Definition at line 76 of file tools/gcp/utils/big_query_utils.py.

create_table()

def big_query_utils.create_table	(		big_query,
			project_id,
			dataset_id,
			table_id,
			table_schema,
			description
	)

Definition at line 65 of file tools/gcp/utils/big_query_utils.py.

create_table2()

def big_query_utils.create_table2	(		big_query,
			project_id,
			dataset_id,
			table_id,
			fields_schema,
			description,
			partition_type = None,
			expiration_ms = None
	)

Definition at line 96 of file tools/gcp/utils/big_query_utils.py.

insert_rows()

def big_query_utils.insert_rows	(		big_query,
			project_id,
			dataset_id,
			table_id,
			rows_list
	)

Definition at line 167 of file tools/gcp/utils/big_query_utils.py.

make_row()

def big_query_utils.make_row	(		unique_row_id,
			row_values_dict
	)

row_values_dict is a dictionary of column name and column value.

Definition at line 201 of file tools/gcp/utils/big_query_utils.py.

patch_table()

def big_query_utils.patch_table	(		big_query,
			project_id,
			dataset_id,
			table_id,
			fields_schema
	)

Definition at line 140 of file tools/gcp/utils/big_query_utils.py.

sync_query_job()

def big_query_utils.sync_query_job	(		big_query,
			project_id,
			query,
			timeout = 5000
	)

Definition at line 187 of file tools/gcp/utils/big_query_utils.py.

Variable Documentation

_EXPIRATION_MS

int big_query_utils._EXPIRATION_MS = 30 * 24 * 60 * 60 * 1000

private

Definition at line 27 of file tools/gcp/utils/big_query_utils.py.

NUM_RETRIES

int big_query_utils.NUM_RETRIES = 3

Definition at line 28 of file tools/gcp/utils/big_query_utils.py.