##Search1

index=_audit sourcetype=audittrail action=search total_run_time=* AND user!=internal_monitoring AND user!=splunk-system-user AND (search_id!="'scheduler*" AND search_id!="scheduler*") | regex search_id="\d{10}\.\d{1,4}"
         | fields total_run_time, search_id, user
         | join type=inner search_id, user
         [ search index=_audit sourcetype=audittrail action=search search=* AND user!=internal_monitoring AND user!=splunk-system-user AND (search_id!="'scheduler*" AND search_id!="scheduler*") AND search!="|history*" AND search!="'typeahead*" AND (search!="'| metadata type=* | search totalCount>0" OR search!="'| metadata type=* | search totalCount > 0") | regex search_id="\d{10}\.\d{1,4}"
                 | fields apiStartTime, apiEndTime, search, search_id, user]
                 | rename apiStartTime as "Time Range Start", apiEndTime as "Time Range End", total_run_time as "Search Duration", user as User, search as Search | timechart max("Search Duration") as "Maximum ad hoc search duration" | appendcols [search index=_internal sourcetype=scheduler
                    | fields _time run_time
                    | timechart max(run_time) as "Scheduled search duration"]




##Search2

index=_introspection sourcetype=splunk_resource_usage data.search_props.sid::* data.search_props.mode!=RT data.search_props.user!="splunk-system-user"
            | eval process = 'data.process'
            | eval args = 'data.args'
            | eval pid = 'data.pid'
            | eval ppid = 'data.ppid'
            | eval elapsed = 'data.elapsed'
            | eval mem_used = 'data.mem_used'
            | eval mem = 'data.mem'
            | eval pct_memory = 'data.pct_memory'
            | eval pct_cpu = 'data.pct_cpu'
            | eval sid = 'data.search_props.sid'
            | eval app = 'data.search_props.app'
            | eval label = 'data.search_props.label'
            | eval type = 'data.search_props.type'
            | eval mode = 'data.search_props.mode'
            | eval user = 'data.search_props.user'
            | eval role = 'data.search_props.role'
            | eval label = if(isnotnull('data.search_props.label'), 'data.search_props.label', "")
            | eval provenance = if(isnotnull('data.search_props.provenance'), 'data.search_props.provenance', "unknown")
            | eval search_head = case(isnotnull('data.search_props.search_head') AND 'data.search_props.role' == "peer", 'data.search_props.search_head', isnull('data.search_props.search_head') AND 'data.search_props.role' == "head", "_self", isnull('data.search_props.search_head') AND 'data.search_props.role' == "peer", "_unknown")
            | eval search_label = if('label'!="", 'label', 'sid')
            | stats max(elapsed) as runtime max(mem_used) as mem_used earliest(_time) as _time by search_label, provenance, type, mode, app, role, user, host
            | eval mem_used = round(mem_used, 2)
            | sort 20 - mem_used, runtime
            | fields search_label, provenance, mem_used, host, runtime, _time, type, mode, app, user, role
            | eval _time=strftime(_time,"%+")
            | rename search_label as Name, provenance as Provenance, mem_used as "Memory Usage (MB)", host as Instance, runtime as Runtime, _time as Started, type as Type, mode as Mode, app as App, user as User, role as Role
            | appendpipe
                [ stats count
                | eval Name="data unavailable"
                | where count==0
                | table Name ]

##Search3

index=_audit sourcetype=audittrail action=search total_run_time=* AND user!=internal_monitoring AND user!=splunk-system-user AND (search_id!="'scheduler*" AND search_id!="scheduler*") | regex search_id="\d{10}\.\d{1,4}"
            | fields total_run_time, search_id, user, result_count, scan_count
            | rename result_count as "Search Result Count", scan_count as "Events Scanned"
            | join type=inner search_id, user
            [ search index=_audit sourcetype=audittrail action=search search=* AND user!=internal_monitoring AND user!=splunk-system-user AND (search_id!="'scheduler*" AND search_id!="scheduler*") AND search!="|history*" AND search!="'typeahead*" AND (search!="'| metadata type=* | search totalCount>0" OR search!="'| metadata type=* | search totalCount > 0") | regex search_id="\d{10}\.\d{1,4}"
            | fields apiStartTime, apiEndTime, search, search_id, user]
            | eval "Search Duration"=tostring(total_run_time, "duration")
            | rename apiStartTime as "Time Range Start", apiEndTime as "Time Range End", user as User, search as Search
            | fieldformat "Events Scanned"=toString('Events Scanned', "commas")
            | eval "Search Time" = strftime(_time, "%Y-%d-%m %H:%M:%S")
            | table "Search Time", User, "Time Range Start", "Time Range End", "Search Duration", "Search Result Count", "Events Scanned", Search
            | sort 20 - "Search Duration"

###Search4

index=_internal sourcetype=scheduler user!=internal_monitoring | table _time user scheduled_time status  run_time result_count savedsearch_name | eval  scheduled_time=strftime(scheduled_time,"%Y-%m-%d %H:%M:%S")
          | sort 20 - run_time
          | eval "Search Time" = strftime(_time, "%Y-%d-%m %H:%M:%S"), "Search Duration"=tostring(total_run_time, "duration")
          | rename scheduled_time as "Scheduled Time", user as "User", savedsearch_name as "Saved Search Name" , status as "Status", result_count as "Search Result Count"
          | fields "Search Time", "User", "Scheduled Time", "Status", "Search Result Count", "Saved Search Name"
          | fields - _time

###Search5

index=_audit sourcetype=audittrail search_id=* NOT(user IN (internal_monitoring splunk-system-user admin)) host=*splunk* action=search
            | transaction search_id
            | eval search=mvfilter(match(search, "^search.*"))
            | where match(search, "^search [^`].*") AND match(info, "completed")
            | eval search_window=round((api_lt-api_et)/60/60/24)
            | eval
              behavior=mvappend(
                if(search_window >= 30, "Long Search Window", null()),
                if(not match(lower(search),"index *(=|::|in)"), "Missing Index", null()),
                if(not match(lower(search),"sourcetype *(=|::|in)"), "Missing Sourcetype",null()),
                if(not match(lower(search),"host *(=|::|in)"), "Missing Host", null())),
              sqs=
                if(search_window >= 30, 7, 0) +
                if(not match(lower(search),"index *(=|::|in)"), 13, 0) +
                if(not match(lower(search),"sourcetype *(=|::|in)"), 3,0) +
                if(not match(lower(search),"host *(=|::|in)"), 2,0)
            | where sqs > 0 AND total_run_time > 10
            | sort - sqs total_run_time
            | table user, search, scan_count, search_window, total_run_time, sqs, behavior
            | eval "Search Duration"=tostring(total_run_time, "duration")
            | rename user as "User", search as "Search SPL", scan_count as "Events Scanned", sqs as "Splunk Query Score", behavior as "Potentially Inefficient Behavior", search_window as "Search Time Range (days)"
            | fieldformat "Events Scanned"=toString('Events Scanned', "commas")
            | fieldformat "Search Time Range (days)"=toString('Search Time Range (days)', "commas")
            | fields "User", "Search SPL", "Events Scanned", "Search Time Range (days)", "Search Duration", "Splunk Query Score", "Potentially Inefficient Behavior"