""""This package contains classes to export metric data"""
import time
import logging
from perfmon.common.export.export import data_exporter
from perfmon.common.utils.locks import FileLock
_log = logging.getLogger(__name__)
# pylint: disable=E0401,W0201,C0301,W0621
[docs]class ExportData(object):
"""This class contains all methods to export dataframe into different
data store types"""
def __init__(self, config, df_dict):
"""Initialise setup"""
self.config = config.copy()
self.df_dict = df_dict
[docs] def get_lock_file(self):
"""Get the lock file to update database exports"""
_log.info('Waiting for the lock file to lock DB')
# Name of the lock file. Prepending with '.' to make it hidden
self.config['db_lock_file'] = '.db_file'
self.fl = FileLock(self.config['db_lock_file'], timeout=10)
while self.fl.lock_exists():
time.sleep(1)
_log.info('Acquired lock file to lock DB')
self.fl.acquire()
[docs] def release_lock(self):
"""Releases lock file"""
_log.info('Releasing DB lock file')
self.fl.release()
[docs] def go(self):
"""Entry point to the class"""
_log.info('Exporting data to file format requested...')
# Acquire lock to update sheet. Multiple jobs might try to update sheet at the same time.
# We need to handle this race condition by file locks
self.get_lock_file()
# Create a pandas dataframe and save the dataframe into several file formats
for export_type in self.config['export']:
data_exporter(export_type, self.config, self.df_dict)
# Once we finish updating the sheet, release lock file
self.release_lock()
if __name__ == '__main__':
import pathlib
# Get project root directory
project_root = pathlib.Path(__file__).parent.parent.parent.parent
# Set GCP service account key file env variable
config = {
'excel_file': 'metrics.xlsx',
'job_id': 1,
'db_file': 'metrics.db',
'h5_file': 'metrics.h5',
'export': ['excel', 'sql', 'hdf5'],
}
df_1 = pandas.DataFrame({'A': [1, 2, 3], 'B': [4, 5, 6]}, index=['a', 'b', 'c'])
df_2 = pandas.Series([1, 2, 3, 4])
df_dict = {
'cpu_metrics': df_1,
'perf_metrics': df_2,
}
export_data = ExportData(config=config, df_dict=df_dict)
export_data.go()