Add zabbix autodiscovery script for rabbitmq

[uio-zabbix.git] / zabbix_elk_container.py

#!/usr/bin/env python
#
# Authors:
# Mustafa Ocak
# muo@uio.no 
#
# Copyright (c) 2016 USIT-University of Oslo
#
# zabbix_elk_container.py: Used by zabbix_agent to pull kibana and logstash
# service information from consul key-value db. 
# Service information from consul is used to low-level discovery of logstash 
# instanses and building url of kibana instanses. 
# Logstash instanses are monitored by net.tcp.service and Kibana web services
# are monitored by this script.  
#
# zabbix_elk_container.py is free software: you can
# redistribute it and/or modify it under the terms of the GNU General
# Public License as published by the Free Software Foundation, either
# version 3 of the License, or (at your option) any later version.
#
# zabbix_elk_container.py is distributed in the hope
# that it will be useful, but WITHOUT ANY WARRANTY; without even the
# implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR
# PURPOSE.  See the GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with sms_send.  If not, see <http://www.gnu.org/licenses/>.
#

import requests
import re
import json
import socket
import sys



# ##########################################################
# Getting the list of all the services registred in consul.
# ##########################################################

def get_services(consul_catalog_url):

    # list of all the registred services in consul
    servicefqdnlist=[]

    try:
        response = requests.get(consul_catalog_url)

        if (response.status_code == 200):
            jsonout=response.json()
            for servicename_fqdn in jsonout:

                # finding all the logstash service fqdn entries. 
                matchobj = re.search(r'logstash',servicename_fqdn,re.IGNORECASE)
                if (matchobj):
                    servicefqdnlist.append(servicename_fqdn)
        return servicefqdnlist
    except Exception as e:
        print e 
        sys.exit(1)

# ##########################################################
# Getting the list of all the logowners registred in consul.
# ##########################################################

def get_logowners (consul_kv_url,servicetype):
    '''
        Get logowners from consul. Consul url structure is as follows:
        http://localhost:8500/v1/kv/uio-elk/services/<servicetype>/<logowner>
    '''
    logowners=[]
    try: 
        response = requests.get(consul_kv_url+servicetype+'?recurse')
        if (response.status_code == 200):
            jsonout = response.json()
            for jsonelement in jsonout:
                
                # jsonelement['Key']= "uio-elk/services/kibana/usit-gid/host"  
                # value[3]="usit-gid"
                value =  jsonelement['Key'].split("/")
                if value[3] not in logowners:
                    logowners.append(value[3])
        return logowners
    except Exception as e:
        print e
        sys.exit(1)

# ######################################################################
# Get list of all logstash instanses for low-level discovery in zabbix.
# ######################################################################

def get_logstash_instanses (consul_kv_url,servicetype,servicefqdnlist,logowners):
    '''
        Pipelines for windows logs are as follows

        --> |nxlog-server|--> |logstash-instanse| --> |ELK| 
        or 
        -->  |logstash-instanse| --> |ELK|

        This function finds whether there is a nxlog-server or not in front of logstash-instanse
        and builds a list of dictionary

        logstash_instanse = {
                    'logowner':..,
                    'type':..,
                    'service':..,
                    'port':..,
                    'host':..,
                    'nxlog':true|false
        }
        logstash_service_list = [ logstash_instanse,...]
    '''
    
    serviceadded=[]
    logstash_lld=[]

    try: 

        for logowner in logowners:
            response = requests.get(consul_kv_url+servicetype+'/'+logowner+'?recurse')
            nxlog = "false"

            if response.status_code == 200:
                jsonout = response.json()
                for jsonelement in jsonout:
                    templist=[]
                    
                    # uio-elk/services/logstash/usit-gsd/tcp/host
                    # value[4] = "tcp"
                      
                    value =  jsonelement['Key'].split("/")
                    host = requests.get(consul_kv_url+servicetype+'/'+logowner+'/'+value[4]+'/host?raw')
                    port = requests.get(consul_kv_url+servicetype+'/'+logowner+'/'+value[4]+'/port?raw')
                    
                    #
                    # Find out logstash instanses which has a nxlog-server in front.
                    # nxlog=true if not false.
                    #

                    for servicefqdn in servicefqdnlist:
                        matchobj = re.search(value[3]+'-nxlog-tcp',servicefqdn,re.IGNORECASE)
                        if matchobj:
                            nxlog="true"
                            fqdn=servicefqdn+".service.consul"
                            templist.append(fqdn)
                    fqdn = host.text+"-"+value[3]+"-"+value[4]+".service.consul"
                    if fqdn not in serviceadded:
                        templist.append(fqdn)
                        
                        logstash_instanse = {'logowner':value[3],'type':value[4],'service':templist,'port':port.text,'host':host.text,'nxlog':nxlog}
                        logstash_lld.append(logstash_instanse)
                    serviceadded = serviceadded+templist
                    templist=[]
                    nxlog="false"

        return logstash_lld

    except Exception as e:
        print e
        sys.exit(1)

# #########################
# Monitor kibana instanses 
# #########################

def monitor_kibana_instanse(kibanaurl):
    '''
    If http response status code is 200 and responce content have 
    key words like "Username" and "Password" then the web service is 
    considered up.
    status
    0 for UP
    1 for DOWN 
    is returned.  
    '''
    status=1
    try: 
        response = requests.get(kibanaurl)
        if (response.status_code == 200):
            if (re.search("username",response.content,re.IGNORECASE)):
                if (re.search("password",response.content,re.IGNORECASE)):
                    status = 0    
                else:
                    status = 1
            else:
                status = 1
        else:
            status = 1
        return status
    except Exception as e:
        print e
        sys.exit(1)
    
def print_usage():
    print "Error: Wrong number of parameters"
    print 'Format: ' + sys.argv[0] + ' logstashlld'
    print 'Format: ' + sys.argv[0] + ' kibanalld'
    print 'Format: ' + sys.argv[0] + ' <kibana_url> monitorkibana'

def print_zabbix_lld_json(lld_data):
    '''
    Print low level discovery json for Zabbix
    '''

    zabbixstr = { "data":lld_data}

    zabbixjson = json.dumps(zabbixstr, indent=4, sort_keys=True)
    print zabbixjson


# ############################################
# Main
# ############################################

if __name__ == '__main__':
     
    consul_catalog_url="http://localhost:8500/v1/catalog/services?recurse"
    consul_kv_url="http://localhost:8500/v1/kv/uio-elk/services/"

    lld_data=[]

    #
    # find which machine the script is running on
    #

    machine = socket.gethostname()
    machinename = machine.split(".")[0]

    if len(sys.argv) == 2:

        # ###############################
        # Logstash low level discovery
        # ###############################

        if sys.argv[1] == "logstashlld":
            servicetype = "logstash"
            servicefqdnlist = get_services(consul_catalog_url)
            logowners = get_logowners(consul_kv_url,servicetype)
            logstash_lld = get_logstash_instanses (consul_kv_url,servicetype,servicefqdnlist,logowners)
            for logstashinstanse in logstash_lld:
                
                if logstashinstanse['nxlog'] == "true":
                    for k in logstashinstanse['service']:
                        matchobj = re.search('nxlog',k,re.IGNORECASE)
                        if matchobj:
                            containerfqdn = k
                    containerport = logstashinstanse['port']
                else:
                    containerfqdn = logstashinstanse['service'][0]
                    containerport = logstashinstanse['port']

                # ##############################################################
                # Get only the containers running on the machine where this 
                # script is run. 
                # ##############################################################

                matchobj = re.search(machinename,containerfqdn,re.IGNORECASE)
                if matchobj:
                    containerdic = {"{#CONTAINER}":containerfqdn,"{#CONTAINER_PORT}":containerport}
                    lld_data.append(containerdic)

            print_zabbix_lld_json(lld_data)

        # ###############################
        # Kibana low level discovery
        # ###############################
   
        elif sys.argv[1] == "kibanalld":

            servicetype = "kibana"
            logowners = get_logowners(consul_kv_url,servicetype)
            for logowner in logowners:
                
                host = requests.get(consul_kv_url+servicetype+'/'+logowner+'/host?raw')
                
                # ##############################################################
                # Get only the kibana containers running on the machine 
                # where this script is run. 
                # ##############################################################

                if (host.text.lower() == machinename.lower()):
                    kibanaurl = "https://"+logowner+".logs.uio.no"
                    kibanadic = {"{#KIBANAURL}":kibanaurl}
                    lld_data.append(kibanadic)

            print_zabbix_lld_json(lld_data)
        else:
            print_usage()

    # ###############################
    # Kibana monitoring
    # ###############################

    elif len(sys.argv) == 3:
        if sys.argv[2] == "monitorkibana":
            kibanaurl = sys.argv[1]
            status=monitor_kibana_instanse(kibanaurl)
            print status
        else:
            print_usage()

    else:
        print_usage()