118 lines
5.0 KiB
Python
118 lines
5.0 KiB
Python
#!/usr/bin/env python3
|
|
|
|
import curator
|
|
import elasticsearch
|
|
import os
|
|
import ssl
|
|
import sys
|
|
|
|
TIMEOUT=120
|
|
|
|
def main():
|
|
if len(sys.argv) != 3:
|
|
print('USAGE: [INDEX_PREFIX=(default "")] [ARCHIVE=(default false)] ... {} NUM_OF_DAYS http://HOSTNAME[:PORT]'.format(sys.argv[0]))
|
|
print('NUM_OF_DAYS ... delete indices that are older than the given number of days.')
|
|
print('HOSTNAME ... specifies which Elasticsearch hosts URL to search and delete indices from.')
|
|
print('TIMEOUT ... number of seconds to wait for master node response.'.format(TIMEOUT))
|
|
print('INDEX_PREFIX ... specifies index prefix.')
|
|
print('INDEX_DATE_SEPARATOR ... specifies index date separator.')
|
|
print('ARCHIVE ... specifies whether to remove archive indices (only works for rollover) (default false).')
|
|
print('ROLLOVER ... specifies whether to remove indices created by rollover (default false).')
|
|
print('ES_USERNAME ... The username required by Elasticsearch.')
|
|
print('ES_PASSWORD ... The password required by Elasticsearch.')
|
|
print('ES_TLS ... enable TLS (default false).')
|
|
print('ES_TLS_CA ... Path to TLS CA file.')
|
|
print('ES_TLS_CERT ... Path to TLS certificate file.')
|
|
print('ES_TLS_KEY ... Path to TLS key file.')
|
|
print('ES_TLS_SKIP_HOST_VERIFY ... (insecure) Skip server\'s certificate chain and host name verification.')
|
|
sys.exit(1)
|
|
|
|
client = create_client(os.getenv("ES_USERNAME"), os.getenv("ES_PASSWORD"), str2bool(os.getenv("ES_TLS", 'false')), os.getenv("ES_TLS_CA"), os.getenv("ES_TLS_CERT"), os.getenv("ES_TLS_KEY"), str2bool(os.getenv("ES_TLS_SKIP_HOST_VERIFY", 'false')))
|
|
ilo = curator.IndexList(client)
|
|
empty_list(ilo, 'Elasticsearch has no indices')
|
|
|
|
prefix = os.getenv("INDEX_PREFIX", '')
|
|
if prefix != '':
|
|
prefix += '-'
|
|
separator = os.getenv("INDEX_DATE_SEPARATOR", '-')
|
|
|
|
if str2bool(os.getenv("ARCHIVE", 'false')):
|
|
filter_archive_indices_rollover(ilo, prefix)
|
|
else:
|
|
if str2bool(os.getenv("ROLLOVER", 'false')):
|
|
filter_main_indices_rollover(ilo, prefix)
|
|
else:
|
|
filter_main_indices(ilo, prefix, separator)
|
|
|
|
empty_list(ilo, 'No indices to delete')
|
|
|
|
for index in ilo.working_list():
|
|
print("Removing", index)
|
|
timeout = int(os.getenv("TIMEOUT", TIMEOUT))
|
|
delete_indices = curator.DeleteIndices(ilo, master_timeout=timeout)
|
|
delete_indices.do_action()
|
|
|
|
|
|
def filter_main_indices(ilo, prefix, separator):
|
|
date_regex = "\d{4}" + separator + "\d{2}" + separator + "\d{2}"
|
|
time_string = "%Y" + separator + "%m" + separator + "%d"
|
|
|
|
ilo.filter_by_regex(kind='regex', value=prefix + "jaeger-(span|service|dependencies)-" + date_regex)
|
|
empty_list(ilo, "No indices to delete")
|
|
# This excludes archive index as we use source='name'
|
|
# source `creation_date` would include archive index
|
|
ilo.filter_by_age(source='name', direction='older', timestring=time_string, unit='days', unit_count=int(sys.argv[1]))
|
|
|
|
|
|
def filter_main_indices_rollover(ilo, prefix):
|
|
ilo.filter_by_regex(kind='regex', value=prefix + "jaeger-(span|service)-\d{6}")
|
|
empty_list(ilo, "No indices to delete")
|
|
# do not remove active write indices
|
|
ilo.filter_by_alias(aliases=[prefix + 'jaeger-span-write'], exclude=True)
|
|
empty_list(ilo, "No indices to delete")
|
|
ilo.filter_by_alias(aliases=[prefix + 'jaeger-service-write'], exclude=True)
|
|
empty_list(ilo, "No indices to delete")
|
|
ilo.filter_by_age(source='creation_date', direction='older', unit='days', unit_count=int(sys.argv[1]))
|
|
|
|
|
|
def filter_archive_indices_rollover(ilo, prefix):
|
|
# Remove only rollover archive indices
|
|
# Do not remove active write archive index
|
|
ilo.filter_by_regex(kind='regex', value=prefix + "jaeger-span-archive-\d{6}")
|
|
empty_list(ilo, "No indices to delete")
|
|
ilo.filter_by_alias(aliases=[prefix + 'jaeger-span-archive-write'], exclude=True)
|
|
empty_list(ilo, "No indices to delete")
|
|
ilo.filter_by_age(source='creation_date', direction='older', unit='days', unit_count=int(sys.argv[1]))
|
|
|
|
|
|
def empty_list(ilo, error_msg):
|
|
try:
|
|
ilo.empty_list_check()
|
|
except curator.NoIndices:
|
|
print(error_msg)
|
|
sys.exit(0)
|
|
|
|
|
|
def str2bool(v):
|
|
return v.lower() in ('true', '1')
|
|
|
|
|
|
def create_client(username, password, tls, ca, cert, key, skipHostVerify):
|
|
context = ssl.create_default_context()
|
|
if ca is not None:
|
|
context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH, cafile=ca)
|
|
elif skipHostVerify:
|
|
context.check_hostname = False
|
|
context.verify_mode = ssl.CERT_NONE
|
|
if username is not None and password is not None:
|
|
return elasticsearch.Elasticsearch(sys.argv[2:], http_auth=(username, password), ssl_context=context)
|
|
elif tls:
|
|
context.load_cert_chain(certfile=cert, keyfile=key)
|
|
return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
|
|
else:
|
|
return elasticsearch.Elasticsearch(sys.argv[2:], ssl_context=context)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|