readClob {
  charset : UTF-8
}

column0,"Look, new hot tub under redwood tree!
All bubbly!",column2

readCSV {
  ...
  quoteChar : "\""

#This is a comment line. It is ignored.

readCSV {
  separator : ","
  columns : [Age,"",Extras,Type]
  ignoreFirstLine : false
  quoteChar : ""
  commentPrefix : ""
  trim : true
  charset : UTF-8
}

readCSV {
  separator : "\t"
  columns : [Age,"",Extras,Type]
  ignoreFirstLine : false
  quoteChar : ""
  commentPrefix : ""
  trim : true
  charset : UTF-8
}

readCSV {
  separator : " "
  columns : [Age,"",Extras,Type]
  ignoreFirstLine : false
  quoteChar : ""
  commentPrefix : ""
  trim : true
  charset : UTF-8
}

readCSV {
  separator : "\u0001" # non-printable CTRL-A character
  columns : [Age,"",Extras,Type]
  ignoreFirstLine : false
  quoteChar : ""
  commentPrefix : ""
  trim : false
  charset : UTF-8
}

readLine {
  ignoreFirstLine : true
  commentPrefix : "#"
  charset : UTF-8
}

# parse log4j with stack traces
readMultiLine {
  regex : "(^.+Exception: .+)|(^\\s+at .+)|(^\\s+\\.\\.\\. \\d+ more)|(^\\s*Caused by:.+)"
  what : previous
  charset : UTF-8
}

# parse sessions; begin new record when we find a line that starts with "Started session"
readMultiLine {
  regex : "Started session.*"
  what : next
  charset : UTF-8
}

addCurrentTime {}

addLocalHost {
  field : my_host
  useIP : false
}

addValues {
  # add values "text/log" and "text/log2" to the source_type output field
  source_type : [text/log, text/log2]

  # add integer 123 to the pid field
  pid : [123]

  # add all values contained in the first_name field to the name field
  name : "@{first_name}"
}

addValuesIfAbsent {
  # add values "text/log" and "text/log2" to the source_type output field
  # unless already present
  source_type : [text/log, text/log2]

  # add integer 123 to the pid field, unless already present
  pid : [123]

  # add all values contained in the first_name field to the name field
  # unless already present
  name : "@{first_name}"
}

callParentPipe {}

# succeed if the _attachment_mimetype field contains a value "avro/binary"
# fail otherwise
contains { _attachment_mimetype : [avro/binary] }

# succeed if the tags field contains a value "version1" or "version2",
# fail otherwise
contains { tags : [version1, version2] }

# convert the timestamp field to "yyyy-MM-dd'T'HH:mm:ss.SSSZ"
# The input may match one of "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
# or "yyyy-MM-dd'T'HH:mm:ss" or "yyyy-MM-dd".
convertTimestamp {
  field : timestamp
  inputFormats : ["yyyy-MM-dd'T'HH:mm:ss.SSS'Z'", "yyyy-MM-dd'T'HH:mm:ss", "yyyy-MM-dd"]
  inputTimezone : America/Los_Angeles
  outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z'"
  outputTimezone : UTC
}

# convert the timestamp field to "yyyy-MM-dd'T'HH:mm:ss.SSSZ"
# and indicate to Solr that it shall round the time down to the current minute
# per http://lucene.apache.org/solr/4_4_0/solr-core/org/apache/solr/util/DateMathParser.html
convertTimestamp {
  ...
  outputFormat : "yyyy-MM-dd'T'HH:mm:ss.SSS'Z/MINUTE'"
  ...
}

decodeBase64 { 
  field : screenshot_base64
}

dropRecord {}

# succeed if the _attachment_mimetype field contains the value "avro/binary"
# and nothing else, fail otherwise
equals { _attachment_mimetype : [avro/binary] }

# succeed if the tags field contains nothing but the values "version1"
# and "highPriority", in that order, fail otherwise
equals { tags : [version1, highPriority] }

extractURIComponents {
  inputField : my_uri
  outputFieldPrefix : uri_component_
}

extractURIComponent {
  inputField : my_uri
  outputField : my_scheme
  component : scheme
}

extractURIQueryParameters {
  parameter : foo
  inputField : myUri
  outputField : my_query_params
}

my_query_params:x
my_query_params:z

findReplace { 
  field : message
  dictionaryFiles : [kite-morphlines-core/src/test/resources/grok-dictionaries]                               
  pattern : """%{WORD:myGroup}"""
  #pattern : """(\b\w+\b)"""      
  isRegex : true
  replacement : "${myGroup}!"
  #replacement : "$1!"
  #replacement : ""
  replaceFirst : false
}

generateUUID {
  field : my_id
}

INT (?:[+-]?(?:[0-9]+))
HOSTNAME \b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)

[+-]?(?:[0-9]+)

\b(?:[0-9A-Za-z][0-9A-Za-z-]{0,62})(?:\.(?:[0-9A-Za-z][0-9A-Za-z-]{0,62}))*(\.?|\b)

expressions : {
  message : """\s+%{INT:pid} %{HOSTNAME:my_name_servers}"""
}

%{REGEX_NAME:GROUP_NAME}

%{INT:pid}

%{HOSTNAME:my_name_servers}

# Index syslog formatted files
#
# Example input line:
#
# <164>Feb  4 10:46:14 syslog sshd[607]: listening on 0.0.0.0 port 22.
#
# Expected output record fields:
#
# syslog_pri:164
# syslog_timestamp:Feb  4 10:46:14
# syslog_hostname:syslog
# syslog_program:sshd
# syslog_pid:607
# syslog_message:listening on 0.0.0.0 port 22.
#
grok {
  dictionaryFiles : [kite-morphlines-core/src/test/resources/grok-dictionaries]
  expressions : {
    message : """<%{POSINT:syslog_pri}>%{SYSLOGTIMESTAMP:syslog_timestamp} %{SYSLOGHOST:syslog_hostname} %{DATA:syslog_program}(?:\[%{POSINT:syslog_pid}\])?: %{GREEDYDATA:syslog_message}"""

    #message2 : "(?<queue_field>.*)"
    #message4 : "%{NUMBER:queue_field}"
  }
}

# Split a line on one or more whitespace into substrings,
# and add the substrings to the "columns" output field.
#
# Example input line with tabs:
#
# "hello\t\tworld\tfoo"
#
# Expected output record fields:
#
# columns:hello
# columns:world
# columns:foo
#
grok {
  expressions : {
    message : """(?<columns>.+?)(\s+|\z)"""
  }
  findSubstrings : true
}

# Index a custom variant of syslog files where subfacility is optional.
#
# Dictionaries in this example are loaded from three places:
# * The my-jar-dictionaries/my-commands file found on the classpath.
# * The local file kite-morphlines-core/src/test/resources/grok-dictionaries.
# * The inline definition shown in dictionaryString.
#
# Example input line:
#
# <179>Jun 10 04:42:51 www.foo.com Jun 10 2013 04:42:51 : %myproduct-3-mysubfacility-123456: Health probe failed
#
# Expected output record fields:
#
# my_message_code:%myproduct-3-mysubfacility-123456
# my_product:myproduct
# my_level:3
# my_subfacility:mysubfacility
# my_message_id:123456
# syslog_message:%myproduct-3-mysubfacility-123456: Health probe failed
#
grok {
  dictionaryResources : [my-jar-dictionaries/my-commands]
  dictionaryFiles : [kite-morphlines-core/src/test/resources/grok-dictionaries]
  dictionaryString : """
    MY_CUSTOM_TIMESTAMP %{MONTH} %{MONTHDAY} %{YEAR} %{TIME}
  """

  expressions : {
    message : """<%{POSINT}>%{SYSLOGTIMESTAMP} %{SYSLOGHOST} %{MY_CUSTOM_TIMESTAMP} : (?<syslog_message>(?<my_message_code>%%{\w+:my_product}-%{\w+:my_level}(-%{\w+:my_subfacility})?-%{\w+:my_message_id}): %{GREEDYDATA})"""
  }
}

# emit only the first 10 records
sample {
  limit : 10
}

if {
  conditions : [
    { contains { _attachment_mimetype : [avro/binary] } }
  ]
  then : [
    { logInfo { format : "processing then..." } }
  ]
  else : [
    { logInfo { format : "processing else..." } }
  ]
}

if {
  conditions : [
    { equals { id : [] } }
  ]
  then : [
    { logTrace { format : "Ignoring record because it has no id: {}", args : ["@{}"] } }
    { dropRecord {} }    
  ]
}

if {
  conditions : [
    { not { equals { malformed : [] } } }
  ]
  then : [
    { logTrace { format : "Ignoring record containing at least one malformed value: {}", args : ["@{}"] } }
    { dropRecord {} }    
  ]
}

public static boolean evaluate(Record record, Config config, Command parent, Command child, MorphlineContext context, Logger logger) {
  // your custom java code block goes here...
}

java {
  imports : "import java.util.*;"
  code: """
    // Update some custom metrics - see http://metrics.codahale.com/getting-started/
    context.getMetricRegistry().counter("myMetrics.myCounter").inc(1);
    context.getMetricRegistry().meter("myMetrics.myMeter").mark(1);
    context.getMetricRegistry().histogram("myMetrics.myHistogram").update(100);
    com.codahale.metrics.Timer.Context timerContext = context.getMetricRegistry().timer("myMetrics.myTimer").time();
                
    // manipulate the contents of a record field
    List tags = record.get("tags");
    if (!tags.contains("hello")) {
      return false;
    }
    tags.add("world");
    
    logger.debug("tags: {} for record: {}", tags, record); // log to SLF4J
    timerContext.stop(); // measure how much time the code block took
    return child.process(record); // pass record to next command in chain
        """
}

# log the entire record at DEBUG level to SLF4J
logDebug { format : "my record: {}", args : ["@{}"] }

# log the timestamp field and the entire record at INFO level to SLF4J
logInfo {
  format : "timestamp: {}, record: {}"
  args : ["@{timestamp}", "@{}"]
}

log4j.logger.org.kitesdk.morphline=TRACE

if {
  conditions : [
    {
      not {
        grok {
          ... some grok expressions go here
        }
      }
    }
  ]
  then : [
    { logDebug { format : "found no grok match: {}", args : ["@{}"] } }
    { dropRecord {} }
  ]
  else : [
    { logDebug { format : "found grok match: {}", args : ["@{}"] } }
  ]
}

pipe {
  id : my_pipe

  # Import all commands in these java packages, subpackages and classes.
  # Other commands on the Java classpath are not visible to this morphline.
  importCommands : [
    "org.kitesdk.**",   # package and all subpackages
    "org.apache.solr.**", # package and all subpackages
    "com.mycompany.mypackage.*", # package only
    "org.kitesdk.morphline.stdlib.GrokBuilder" # fully qualified class
  ]

  commands : [
    { addValues { foo : bar }}
    { logDebug { format : "output record: {}", args : ["@{}"] } }
  ]
}

sample {
  probability : 0.0001
  seed : 12345
}

separateAttachments {}

setValues {
  # assign values "text/log" and "text/log2" to source_type output field
  source_type : [text/log, text/log2]

  # assign the integer 123 to the pid field
  pid : [123]

  # remove the url field
  url : []

  # assign all values contained in the first_name field to the name field
  name : "@{first_name}"
}

split { 
  inputField : message
  outputFields : [first_name, last_name, "", age]          
  separator : ","        
  isRegex : false      
  #separator : """\s*,\s*"""        
  #isRegex : true      
  addEmptyStrings : false
  trim : true          
}

message:"Nadja,Redwood,female,8"

first_name:Nadja
last_name:Redwood
age:8

split { 
  inputField : message
  outputField : substrings          
  separator : ","        
  isRegex : false      
  #separator : """\s*,\s*"""        
  #isRegex : true      
  addEmptyStrings : false
  trim : true          
}

message:"_a ,_b_ ,c__"

substrings:_a
substrings:_b_
substrings:c__

split { 
  inputField : message
  outputField : substrings
  # dictionaryFiles : [kite-morphlines-core/src/test/resources/grok-dictionaries] 
  dictionaryString : """COMMA_SURROUNDED_BY_WHITESPACE \s*,\s*"""          
  separator : """%{COMMA_SURROUNDED_BY_WHITESPACE}"""        
  # separator : """\s*,\s*"""        
  isRegex : true      
  addEmptyStrings : true
  trim : false          
}

splitKeyValue { 
  inputField : params
  separator : "="
  outputFieldPrefix : "/"
}

params:foo=x
params: foo = y 
params:foo 
params:fragment=z

/foo:x
/foo:y
/fragment:z

# read each line in the file
{ 
  readLine {
    charset : UTF-8
  }
} 
                
# extract timestamp and key value pair string
{ 
  grok { 
    dictionaryFiles : [target/test-classes/grok-dictionaries/grok-patterns]
    expressions : { 
      message : """%{SYSLOGTIMESTAMP:timestamp} %{GREEDYDATA:key_value_pairs_string}"""
    }
  }
}

# split key value pair string on blanks into an array of key value pairs
{ 
  split { 
    inputField : key_value_pairs_string
    outputField : key_value_array          
    separator : " "        
  }
}

# split each key value pair on '=' char and extract its value into record fields named after the key
{ 
  splitKeyValue { 
    inputField : key_value_array
    outputFieldPrefix : ""          
    separator : "="        
    addEmptyStrings : false
    trim : true          
  }
}

# remove temporary work fields
{ 
  setValues {
    key_value_pairs_string : []
    key_value_array : []
  }
}

Feb  6 12:04:42 IN=eth1 OUT=eth0 SRC=1.2.3.4 DST=6.7.8.9 ACK DF WINDOW=0

timestamp:Feb  6 12:04:42
IN:eth1
OUT:eth0
SRC:1.2.3.4
DST:6.7.8.9
WINDOW:0

startReportingMetricsToCSV {
  outputDir : "mytest/metricsLogs"
  frequency : "10 seconds"
  locale : en_US
}

startReportingMetricsToCSV {
  outputDir : "mytest/metricsLogs"
  frequency : "10 seconds"
  locale : en_US
  defaultDurationUnit : milliseconds
  defaultRateUnit : seconds
  metricFilter : {
    includes : { # if absent defaults to match all
      "literal:foo" : "glob:foo*"
      "regex:.*" : "glob:*"
    }
    excludes : { # if absent defaults to match none
      "literal:foo.bar" : "*"
    }
  }          
}

t,count,mean_rate,m1_rate,m5_rate,m15_rate,rate_unit
1380054913,2,409.752100,0.000000,0.000000,0.000000,events/second
1380055913,2,258.131131,0.000000,0.000000,0.000000,events/second

startReportingMetricsToJMX {
  domain : myMetrics
}

startReportingMetricsToJMX {
  domain : myMetrics
  durationUnits : {
    myMetrics.myTimer : minutes
  }
  defaultDurationUnit : milliseconds
  rateUnits : {
    myMetrics.myTimer : milliseconds
    morphline.logDebug.numProcessCalls : milliseconds
  }
  defaultRateUnit : seconds
  metricFilter : {
    includes : { # if absent defaults to match all
      "literal:foo" : "glob:foo*"
      "regex:.*" : "glob:*"
    }
    excludes : { # if absent defaults to match none
      "literal:foo.bar" : "*"
    }
  }          
}

startReportingMetricsToSLF4J {
  logger : "org.kitesdk.morphline.domain1"
  frequency : "10 seconds"
}

startReportingMetricsToSLF4J {
  logger : "org.kitesdk.morphline.domain1"
  frequency : "10 seconds"
  defaultDurationUnit : milliseconds
  defaultRateUnit : seconds
  metricFilter : {
    includes : { # if absent defaults to match all
      "literal:foo" : "glob:foo*"
      "regex:.*" : "glob:*"
    }
    excludes : { # if absent defaults to match none
      "literal:foo.bar" : "*"
    }
  }          
}

457  [metrics-logger-reporter-thread-1] INFO  org.kitesdk.morphline.domain1  - type=METER, name=morphline.logDebug.numProcessCalls, count=2, mean_rate=144.3001443001443, m1=0.0, m5=0.0, m15=0.0, rate_unit=events/second

toByteArray { field : _attachment_body }

toString { field : source_type }

translate {
  field : level
  dictionary : {
     0 : Emergency
     1 : Alert
     2 : Critical
     3 : Error
     4 : Warning
     5 : Notice
     6 : Informational
     7 : Debug
  }
  fallback : Unknown # if no fallback is defined and no match is found then the command fails
}

tryRules {
  catchExceptions : false
  throwExceptionIfAllRulesFailed : true
  rules : [
    # next rule of tryRules cmd:
    {
      commands : [
        { contains { _attachment_mimetype : [avro/binary] } }
        ... handle Avro data here
        { logDebug { format : "output record: {}", args : ["@{}"] } }
      ]
    }

    # next rule of tryRules cmd:
    {
      commands : [
        { contains { _attachment_mimetype : [text/csv] } }
        ... handle CSV data here
        { logDebug { format : "output record: {}", args : ["@{}"] } }
      ]
    }
    
    # if desired, the last rule can serve as a fallback mechanism 
    # for records that don't match any rule:
    {
      commands : [
        { logWarn { format : "Ignoring record with unsupported input format: {}", args : ["@{}"] } }
        { dropRecord {} }    
      ]
    }
  ]
}

# Parse Avro container file and emit a record for each avro object
readAvroContainer {
  # Optionally, require the input to match one of these MIME types:
  # supportedMimeTypes : [avro/binary]

  # Optionally, use this Avro schema in JSON format inline for reading:
  # readerSchemaString : """<json can go here>"""

  # Optionally, use this Avro schema file in JSON format for reading:
  # readerSchemaFile : /path/to/syslog.avsc
}

# Parse Avro and emit a record for each avro object
readAvro {
  # supportedMimeTypes : [avro/binary]
  # readerSchemaString : """<json can go here>"""
  # readerSchemaFile : test-documents/sample-statuses-20120906-141433-subschema.avsc
  # writerSchemaString : """<json can go here>"""
  writerSchemaFile : test-documents/sample-statuses-20120906-141433.avsc
}

extractAvroTree {
  outputFieldPrefix : ""
}

extractAvroPaths {
  flatten : true
  paths : {
    my_price : /price

    my_docId : /docId
    my_links : /links
    my_links_backward : "/links/backward"
    my_links_forward : "/links/forward"
    my_name_language_code : "/name[]/language[]/code"
    my_name_language_country : "/name[]/language[]/country"
    my_name : /name

    /mymapField/foo/label : /mapField/foo/label/
  }
}

{ 
  readAvroContainer { }
}

{
  java { 
    imports : """
      import org.apache.avro.generic.GenericRecord;
      import org.kitesdk.morphline.base.Fields;
      // import com.cloudera.cdk.morphline.base.Fields; // use this for CDK
    """
    code : """
      GenericRecord root = (GenericRecord) record.getFirstValue(Fields.ATTACHMENT_BODY);
      GenericRecord links = (GenericRecord) root.get("links"); // traverse via Avro Tree API
      String forwardLinks = links.get("forward").toString(); // traverse via Avro Tree API
      record.put("forwardLinks", forwardLinks);
      logger.debug("My output record: {}", record);
      return child.process(record);
    """
    } 
  }
}

toAvro {
  #schemaFile : /path/to/interop.avsc 
  #schemaField : _dataset_descriptor_schema 
  schemaString : """
    {
      "type" : "record",
      "name" : "Rating",
      "fields" : [ 
        {
          "name" : "userId",
          "type" : "int"
        }, 
        {
          "name" : "rating",
          "type" : ["int","null"]
        }, 
        {
          "name" : "reviews",
          "type" : {"type": "array", "items": "string"}
        }, 
        {
          "name" : "history",
          "type" : ["null", {"type": "map", "values":     
                              {"type": "record", "name": "Foo",
                                   "fields": [{"name": "timestamp", "type": "long"}]}}]
        } 
      ]
    }
  """
  mappings : { 
    userId : morphlineUserId
  }          
} 

writeAvroToByteArray {
  format : container 
  codec : snappy
}

readJson {}

readJson {
  outputClass : java.util.Map
}

extractJsonPaths {
  flatten : true
  paths : {
    my_price : /price

    my_docId : /docId
    my_links : /links
    my_links_backward : "/links/backward"
    my_links_forward : "/links/forward"
    my_name_language_code : "/name[]/language[]/code"
    my_name_language_country : "/name[]/language[]/country"
    my_name : /name    
  }
}

{ 
  readJson { }
}

{
  java { 
    imports : """
      import com.fasterxml.jackson.databind.JsonNode;
      import org.kitesdk.morphline.base.Fields;
      // import com.cloudera.cdk.morphline.base.Fields; // use this for CDK
    """
    code : """
      JsonNode rootNode = (JsonNode) record.getFirstValue(Fields.ATTACHMENT_BODY);
      String forwardLinks = rootNode.get("links").get("forward").asText(); // traverse via Jackson Tree API
      record.put("forwardLinks", forwardLinks);
      logger.debug("My output record: {}", record);
      return child.process(record);
    """
    } 
  }
}

downloadHdfsFile {
  inputFiles : ["hdfs://c2202.mycompany.com/user/foo/configs/sample-schema.avsc"]
  outputDir : "myconfigs"
}

readAvroParquetFile {
  # Optionally, use this Avro schema in JSON format inline for projection:
  # projectionSchemaString : """<json can go here>"""

  # Optionally, use this Avro schema file in JSON format for projection:
  # projectionSchemaFile : /path/to/syslog.avsc
}

readRCFile {
  readMode: row
  includeMetaData: false
  columns: [
    {
      inputField: 0
      outputField: name
      writableClass: "org.apache.hadoop.io.Text"
    }
    {
      inputField: 3
      outputField: age
      writableClass: "org.apache.hadoop.io.LongWritable"
    }
    {
      inputField: 1000
      outputField: photo
      writableClass: "org.apache.hadoop.hive.serde2.columnar.BytesRefWritable"
    }
  ]
}

readSequenceFile {
  keyField : "key"
  valueField : "value"
}

# extract geolocation info into a Jackson JsonNode Java object 
# and store it into the _attachment_body field:
geoIP {
  inputField : ip
  database : "target/test-classes/GeoLite2-City.mmdb"
}

# extract parts of the geolocation info from the Jackson JsonNode Java 
# object contained in the _attachment_body field and store the parts in
# the given record output fields:      
extractJsonPaths {
  flatten : false
  paths : { 
    /country/iso_code : /country/iso_code
    /country/names/en : /country/names/en
    /country/names/zh-CN : /country/names/zh-CN       
    "/subdivisions[]/names/en" : "/subdivisions[]/names/en"     
    "/subdivisions[]/iso_code" : "/subdivisions[]/iso_code"     
    /city/names/en : /city/names/en
    /postal/code : /postal/code
    /location/latitude : /location/latitude
    /location/longitude : /location/longitude
    /location/latitude_longitude : /location/latitude_longitude
    /location/longitude_latitude : /location/longitude_latitude
  } 
}

ip: 128.101.101.101
/country/iso_code: US
/country/names/en: United States
/country/names/zh-CN: 美国
/subdivisions[]/names/en: Minnesota
/subdivisions[]/iso_code: MN
/city/names/en: Minneapolis
/postal/code: 55455
/location/latitude: 44.9733
/location/longitude: -93.2323
/location/latitude_longitude: 44.9733,-93.2323
/location/longitude_latitude: -93.2323,44.9733

{
   "city":{
      "geoname_id":5037649,
      "names":{
         "de":"Minneapolis",
         "en":"Minneapolis",
         "es":"Mineápolis",
         "fr":"Minneapolis",
         "ja":"ミネアポリス",
         "pt-BR":"Minneapolis",
         "ru":"Миннеаполис",
         "zh-CN":"明尼阿波利斯"
      }
   },
   "continent":{
      "code":"NA",
      "geoname_id":6255149,
      "names":{
         "de":"Nordamerika",
         "en":"North America",
         "es":"Norteamérica",
         "fr":"Amérique du Nord",
         "ja":"北アメリカ",
         "pt-BR":"América do Norte",
         "ru":"Северная Америка",
         "zh-CN":"北美洲"
      }
   },
   "country":{
      "geoname_id":6252001,
      "iso_code":"US",
      "names":{
         "de":"USA",
         "en":"United States",
         "es":"Estados Unidos",
         "fr":"États-Unis",
         "ja":"アメリカ合衆国",
         "pt-BR":"Estados Unidos",
         "ru":"США",
         "zh-CN":"美国"
      }
   },
   "location":{
      "latitude":44.9733,
      "longitude":-93.2323,
      "metro_code":"613",
      "time_zone":"America/Chicago"
      "latitude_longitude":"44.9733,-93.2323"
      "longitude_latitude":"-93.2323,44.9733"
   },
   "postal":{
      "code":"55455"
   },
   "registered_country":{
      "geoname_id":6252001,
      "iso_code":"US",
      "names":{
         "de":"USA",
         "en":"United States",
         "es":"Estados Unidos",
         "fr":"États-Unis",
         "ja":"アメリカ合衆国",
         "pt-BR":"Estados Unidos",
         "ru":"США",
         "zh-CN":"美国"
      }
   },
   "subdivisions":[
      {
         "geoname_id":5037779,
         "iso_code":"MN",
         "names":{
            "en":"Minnesota",
            "es":"Minnesota",
            "ja":"ミネソタ州",
            "ru":"Миннесота"
         }
      }
   ]
}

registerJVMMetrics {}

startReportingMetricsToHTTP {
  port : 8080
}

pong

{"deadlocks":{"healthy":true}}

{
 "version" : "3.0.0",
 "gauges" : {
   "jvm.gc.ConcurrentMarkSweep.count" : {
     "value" : 0
   },
   "jvm.gc.ConcurrentMarkSweep.time" : {
     "value" : 0
   },
   "jvm.gc.ParNew.count" : {
     "value" : 4
   },
   "jvm.gc.ParNew.time" : {
     "value" : 29
   },
   "jvm.memory.heap.committed" : {
     "value" : 85000192
   },
   "jvm.memory.heap.init" : {
     "value" : 0
   },
   "jvm.memory.heap.max" : {
     "value" : 129957888
   },
   "jvm.memory.heap.usage" : {
     "value" : 0.1319703810514372
   },
   "jvm.memory.heap.used" : {
     "value" : 17150592
   },
   "jvm.memory.non-heap.committed" : {
     "value" : 24711168
   },
   "jvm.memory.non-heap.init" : {
     "value" : 24317952
   },
   "jvm.memory.non-heap.max" : {
     "value" : 136314880
   },
   "jvm.memory.non-heap.usage" : {
     "value" : 0.16856530996469352
   },
   "jvm.memory.non-heap.used" : {
     "value" : 22978104
   },
   "jvm.memory.pools.CMS-Old-Gen.usage" : {
     "value" : 0.05705025643464222
   },
   "jvm.memory.pools.CMS-Perm-Gen.usage" : {
     "value" : 0.25629341311571074
   },
   "jvm.memory.pools.Code-Cache.usage" : {
     "value" : 0.018703460693359375
   },
   "jvm.memory.pools.Par-Eden-Space.usage" : {
     "value" : 0.5095581972509399
   },
   "jvm.memory.pools.Par-Survivor-Space.usage" : {
     "value" : 0.9115804036458334
   },
   "jvm.memory.total.committed" : {
     "value" : 109711360
   },
   "jvm.memory.total.init" : {
     "value" : 24317952
   },
   "jvm.memory.total.max" : {
     "value" : 266272768
   },
   "jvm.memory.total.used" : {
     "value" : 40248344
   },
   "jvm.threads.blocked.count" : {
     "value" : 2
   },
   "jvm.threads.count" : {
     "value" : 22
   },
   "jvm.threads.daemon.count" : {
     "value" : 4
   },
   "jvm.threads.deadlocks" : {
     "value" : [ ]
   },
   "jvm.threads.new.count" : {
     "value" : 0
   },
   "jvm.threads.runnable.count" : {
     "value" : 10
   },
   "jvm.threads.terminated.count" : {
     "value" : 0
   },
   "jvm.threads.timed_waiting.count" : {
     "value" : 8
   },
   "jvm.threads.waiting.count" : {
     "value" : 2
   }
 },
 "counters" : {
   "myMetrics.myCounter" : {
     "count" : 1
   }
 },
 "histograms" : {
   "myMetrics.myHistogram" : {
     "count" : 1,
     "max" : 100,
     "mean" : 100.0,
     "min" : 100,
     "p50" : 100.0,
     "p75" : 100.0,
     "p95" : 100.0,
     "p98" : 100.0,
     "p99" : 100.0,
     "p999" : 100.0,
     "stddev" : 0.0
   }
 },
 "meters" : {
   "morphline.java.numNotifyCalls" : {
     "count" : 1,
     "m15_rate" : 0.16929634497812282,
     "m1_rate" : 0.19779007785878447,
     "m5_rate" : 0.1934432200964012,
     "mean_rate" : 0.06666243138019297,
     "units" : "events/second"
   },
   "morphline.java.numProcessCalls" : {
     "count" : 1,
     "m15_rate" : 0.16929634497812282,
     "m1_rate" : 0.19779007785878447,
     "m5_rate" : 0.1934432200964012,
     "mean_rate" : 0.06666191145031655,
     "units" : "events/second"
   },
   "morphline.logDebug.numNotifyCalls" : {
     "count" : 3,
     "m15_rate" : 0.5078890349343685,
     "m1_rate" : 0.5933702335763534,
     "m5_rate" : 0.5803296602892035,
     "mean_rate" : 0.1999690981087056,
     "units" : "events/second"
   },
   "morphline.logDebug.numProcessCalls" : {
     "count" : 3,
     "m15_rate" : 0.5078890349343685,
     "m1_rate" : 0.5933702335763534,
     "m5_rate" : 0.5803296602892035,
     "mean_rate" : 0.19996765856402157,
     "units" : "events/second"
   },
   "morphline.logWarn.numNotifyCalls" : {
     "count" : 2,
     "m15_rate" : 0.33859268995624564,
     "m1_rate" : 0.39558015571756894,
     "m5_rate" : 0.3868864401928024,
     "mean_rate" : 0.11979779569659962,
     "units" : "events/second"
   },
   "morphline.logWarn.numProcessCalls" : {
     "count" : 2,
     "m15_rate" : 0.33859268995624564,
     "m1_rate" : 0.39558015571756894,
     "m5_rate" : 0.3868864401928024,
     "mean_rate" : 0.11979702071997292,
     "units" : "events/second"
   },
   "morphline.pipe.numNotifyCalls" : {
     "count" : 1,
     "m15_rate" : 0.16929634497812282,
     "m1_rate" : 0.19779007785878447,
     "m5_rate" : 0.1934432200964012,
     "mean_rate" : 0.05774150456461029,
     "units" : "events/second"
   },
   "morphline.pipe.numProcessCalls" : {
     "count" : 1,
     "m15_rate" : 0.16929634497812282,
     "m1_rate" : 0.19779007785878447,
     "m5_rate" : 0.1934432200964012,
     "mean_rate" : 0.05766282424671017,
     "units" : "events/second"
   },
   "morphline.registerJVMMetrics.numNotifyCalls" : {
     "count" : 1,
     "m15_rate" : 0.16929634497812282,
     "m1_rate" : 0.19779007785878447,
     "m5_rate" : 0.1934432200964012,
     "mean_rate" : 0.059902898599428295,
     "units" : "events/second"
   },
   "morphline.registerJVMMetrics.numProcessCalls" : {
     "count" : 1,
     "m15_rate" : 0.16929634497812282,
     "m1_rate" : 0.19779007785878447,
     "m5_rate" : 0.1934432200964012,
     "mean_rate" : 0.059902518235973874,
     "units" : "events/second"
   },
   "morphline.startReportingMetricsToHTTP.numNotifyCalls" : {
     "count" : 3,
     "m15_rate" : 0.5078890349343685,
     "m1_rate" : 0.5933702335763534,
     "m5_rate" : 0.5803296602892035,
     "mean_rate" : 0.19066147711547488,
     "units" : "events/second"
   },
   "morphline.startReportingMetricsToHTTP.numProcessCalls" : {
     "count" : 3,
     "m15_rate" : 0.5078890349343685,
     "m1_rate" : 0.5933702335763534,
     "m5_rate" : 0.5803296602892035,
     "mean_rate" : 0.19066014422550162,
     "units" : "events/second"
   },
   "myMetrics.myMeter" : {
     "count" : 1,
     "m15_rate" : 0.18400888292586468,
     "m1_rate" : 0.19889196960097935,
     "m5_rate" : 0.1966942907643235,
     "mean_rate" : 0.0698670918312095,
     "units" : "events/second"
   }
 },
 "timers" : {
   "myMetrics.myTimer" : {
     "count" : 1,
     "max" : 1.4000000000000001E-5,
     "mean" : 1.4000000000000001E-5,
     "min" : 1.4000000000000001E-5,
     "p50" : 1.4000000000000001E-5,
     "p75" : 1.4000000000000001E-5,
     "p95" : 1.4000000000000001E-5,
     "p98" : 1.4000000000000001E-5,
     "p99" : 1.4000000000000001E-5,
     "p999" : 1.4000000000000001E-5,
     "stddev" : 0.0,
     "m15_rate" : 0.18400888292586468,
     "m1_rate" : 0.19889196960097935,
     "m5_rate" : 0.1966942907643235,
     "mean_rate" : 0.0698417274708746,
     "duration_units" : "seconds",
     "rate_units" : "calls/second"
   },
   "myMetrics.myTimer2" : {
     "count" : 1,
     "max" : 0.0,
     "mean" : 0.0,
     "min" : 0.0,
     "p50" : 0.0,
     "p75" : 0.0,
     "p95" : 0.0,
     "p98" : 0.0,
     "p99" : 0.0,
     "p999" : 0.0,
     "stddev" : 0.0,
     "m15_rate" : 0.18400888292586468,
     "m1_rate" : 0.19889196960097935,
     "m5_rate" : 0.1966942907643235,
     "mean_rate" : 0.0698418152725891,
     "duration_units" : "seconds",
     "rate_units" : "calls/second"
   }
 }
}

main id=1 state=TIMED_WAITING
    at java.lang.Thread.sleep(Native Method)
    at org.kitesdk.morphline.metrics.servlets.HttpMetricsMorphlineTest.testBasic(HttpMetricsMorphlineTest.java:51)
    at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
    at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:39)
    at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:25)
    at java.lang.reflect.Method.invoke(Method.java:597)
    at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:45)
    at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:15)
    at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:42)
    at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:20)
    at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:28)
    at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:30)
    at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:263)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:68)
    at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:47)
    at org.junit.runners.ParentRunner$3.run(ParentRunner.java:231)
    at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:60)
    at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:229)
    at org.junit.runners.ParentRunner.access$000(ParentRunner.java:50)
    at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:222)
    at org.junit.runners.ParentRunner.run(ParentRunner.java:300)
    at org.eclipse.jdt.internal.junit4.runner.JUnit4TestReference.run(JUnit4TestReference.java:50)
    at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:38)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:467)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:683)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:390)
    at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:197)

Reference Handler id=2 state=WAITING
    - waiting on <0x7418e252> (a java.lang.ref.Reference$Lock)
    - locked <0x7418e252> (a java.lang.ref.Reference$Lock)
    at java.lang.Object.wait(Native Method)
    at java.lang.Object.wait(Object.java:485)
    at java.lang.ref.Reference$ReferenceHandler.run(Reference.java:116)

... and so on

detectMimeType {
  includeDefaultMimeTypes : false
  #mimeTypesFiles : [src/test/resources/custom-mimetypes.xml]
  mimeTypesString :
    """
    <mime-info>
      <mime-type type="text/space-separated-values">
        <glob pattern="*.ssv"/>
      </mime-type>

      <mime-type type="avro/binary">
        <magic priority="50">
          <match value="0x4f626a01" type="string" offset="0"/>
        </magic>
        <glob pattern="*.avro"/>
      </mime-type>

      <mime-type type="mytwittertest/json+delimited+length">
        <magic priority="50">
          <match value="[0-9]+(\r)?\n\\{&quot;" type="regex" offset="0:16"/>
        </magic>
      </mime-type>
    </mime-info>
    """
}

decompress {}

unpack {}

convertHTML {
  charset : UTF-8
}

xquery {
  fragments : [
    {
      fragmentPath : "/"
      externalVariables : {
        myVariable : "hello world"
      }            
      queryString : """
        (: Example test xquery :)        
        
        declare variable $myVariable as xs:string external;
        
        for $tweet in /tweets/tweet
        return 
        <user>
          {$tweet/@id} 
          {$tweet/user/@screen_name} 
          <myStatusCounts>{string($tweet/user/@statuses_count)}</myStatusCounts>
          <text>{$tweet/text}</text>
          <greeting>{$myVariable}</greeting>
        </user>
      """
    }
  ]
}

id:11111112
screen_name:fake_user1
myStatusCounts:11111
text:Come, see new hot tub under Redwood Tree!
greeting:hello world

xquery {
  fragments : [
    {
      fragmentPath : "/"
      queryString : """
        (: Example xquery :)
        for $req in /request
        return
          <fieldsToIndex>
            <date> { string($req/data/agreementDate) } </date>
            <tradeId> { string($req/trade/@tradeId) } </tradeId>              
            <partyId>
              {
                for $keyword in $req/trade/keyWords/keyword 
                  where $keyword/name = "memberId"
                    return string($keyword/value)                  
              }
            </partyId>
            <fullText> { $req } </fullText>
        </fieldsToIndex>
      """
    }
  ]
}

xslt {
  fragments : [
    {
      fragmentPath : "/"
      parameters : {
        myVariable : "hello world"
      }            
      queryString : """
        <!-- Example XSLT identity transformation -->
        <xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform" version="1.0">
        
        <xsl:template match="@*|node()">
          <xsl:copy>
            <xsl:apply-templates select="@*|node()"/>
          </xsl:copy>
        </xsl:template>
        
        </xsl:stylesheet>
      """
    }
  ]
}

solrLocator : {
  # Name of solr collection
  collection : collection1

  # ZooKeeper ensemble
  zkHost : "127.0.0.1:2181/solr"

  # Max number of documents to pass per RPC from morphline to Solr Server
  # batchSize : 1000
}

loadSolr {
  solrLocator : {
    # Name of solr collection
    collection : collection1

    # ZooKeeper ensemble
    zkHost : "127.0.0.1:2181/solr"

    # Max number of docs to pass per RPC from morphline to Solr Server
    # batchSize : 1000
  }
  boosts : {
    id : 2.0 # assign to the id field a boost value 2.0
  }
}

generateSolrSequenceKey {
  baseIdField: ignored_base_id
  solrLocator : ${SOLR_LOCATOR}
}

sanitizeUnknownSolrFields {
  solrLocator : ${SOLR_LOCATOR}
}

tokenizeText {
  inputField : message
  outputField : tokens
  solrFieldType : text_en
  solrLocator : {
    # Name of solr collection
    collection : collection1

    # ZooKeeper ensemble
    zkHost : "127.0.0.1:2181/solr"
    
    # solrHomeDir : "example/solr/collection1"    
  }
}

tokens:hello
tokens:world
tokens:foo
tokens:bar.com
tokens:123

...
<fieldType name="text_en" class="solr.TextField" positionIncrementGap="100">
  <analyzer type="index">
    <tokenizer class="solr.StandardTokenizerFactory"/>
    <filter class="solr.StopFilterFactory"
       ignoreCase="true"
       words="lang/stopwords_en.txt"
       enablePositionIncrements="true"
    />
    <filter class="solr.LowerCaseFilterFactory"/>
    <filter class="solr.EnglishPossessiveFilterFactory"/>
    <filter class="solr.KeywordMarkerFilterFactory" protected="protwords.txt"/>
    <filter class="solr.PorterStemFilterFactory"/>
  </analyzer>
</fieldType>

solrCell {
  solrLocator : ${SOLR_LOCATOR}

  # extract some fields
  capture : [content, a, h1, h2]

  # rename exif_image_height field to text field
  # rename a field to anchor field
  # rename h1 field to heading1 field
  fmap : { exif_image_height : text, a : anchor, h1 : heading1 }

  # xpath : "/xhtml:html/xhtml:body/xhtml:div/descendant:node()"

  parsers : [ # one or more nested Tika parsers
    { parser : org.apache.tika.parser.jpeg.JpegParser }
  ]
}

morphlines : [
  {
    id : morphline1
    importCommands : ["org.kitesdk.**", "org.apache.solr.**"]

    commands : [
      {
        # emit one output record for each attachment in the input
        # record's list of attachments. The result is a list of
        # records, each of which has at most one attachment.
        separateAttachments {}
      }

      {
        # used for auto-detection if MIME type isn't explicitly supplied
        detectMimeType {
          includeDefaultMimeTypes : true
          mimeTypesFiles : [target/test-classes/custom-mimetypes.xml]
        }
      }

      {
        tryRules {
          throwExceptionIfAllRulesFailed : true
          rules : [
            # next rule of tryRules cmd:
            {
              commands : [
                { logDebug { format : "hello unpack" } }
                { unpack {} }
                { generateUUID {} }
                { callParentPipe {} }
              ]
            }

            # next rule of tryRules cmd:
            {
              commands : [
                { logDebug { format : "hello decompress" } }
                { decompress {} }
                { callParentPipe {} }
              ]
            }

            # next rule of tryRules cmd:
            {
              commands : [
                {
                  readAvroContainer {
                    supportedMimeTypes : [avro/binary]
                    # optional, avro json schema blurb for getSchema()
                    # readerSchemaString : "<json can go here>"
                    # readerSchemaFile : /path/to/syslog.avsc
                  }
                }

                { extractAvroTree {} }

                {
                  setValues {
                    id : "@{/id}"
                    user_screen_name : "@{/user_screen_name}"
                    text : "@{/text}"
                  }
                }

                {
                  sanitizeUnknownSolrFields {
                    solrLocator : ${SOLR_LOCATOR}
                  }
                }
              ]
            }

            # next rule of tryRules cmd:
            {
              commands : [
                {
                  readJsonTestTweets {
                    supportedMimeTypes : ["mytwittertest/json+delimited+length"]
                  }
                }

                {
                  sanitizeUnknownSolrFields {
                    solrLocator : ${SOLR_LOCATOR}
                  }
                }
              ]
            }

            # next rule of tryRules cmd:
            {
              commands : [
                { logDebug { format : "hello solrcell" } }
                {
                  # wrap SolrCell around an Tika parsers
                  solrCell {
                    solrLocator : ${SOLR_LOCATOR}

                    capture : [
                      # twitter feed schema
                      user_friends_count
                      user_location
                      user_description
                      user_statuses_count
                      user_followers_count
                      user_name
                      user_screen_name
                      created_at
                      text
                      retweet_count
                      retweeted
                      in_reply_to_user_id
                      source
                      in_reply_to_status_id
                      media_url_https
                      expanded_url
                     ]

                    # rename "content" field to "text" fields
                    fmap : { content : text, content-type : content_type }

                    lowernames : true

                    # Tika parsers to be registered:
                    parsers : [
                      # { parser : org.apache.tika.parser.AutoDetectParser }
                      { parser : org.apache.tika.parser.asm.ClassParser }
                      { parser : org.gagravarr.tika.FlacParser }
                      { parser : org.apache.tika.parser.audio.AudioParser }
                      { parser : org.apache.tika.parser.audio.MidiParser }
                      { parser : org.apache.tika.parser.crypto.Pkcs7Parser }
                      { parser : org.apache.tika.parser.dwg.DWGParser }
                      { parser : org.apache.tika.parser.epub.EpubParser }
                      { parser : org.apache.tika.parser.executable.ExecutableParser }
                      { parser : org.apache.tika.parser.feed.FeedParser }
                      { parser : org.apache.tika.parser.font.AdobeFontMetricParser }
                      { parser : org.apache.tika.parser.font.TrueTypeParser }
                      { parser : org.apache.tika.parser.xml.XMLParser }
                      { parser : org.apache.tika.parser.html.HtmlParser }
                      { parser : org.apache.tika.parser.image.ImageParser }
                      { parser : org.apache.tika.parser.image.PSDParser }
                      { parser : org.apache.tika.parser.image.TiffParser }
                      { parser : org.apache.tika.parser.iptc.IptcAnpaParser }
                      { parser : org.apache.tika.parser.iwork.IWorkPackageParser }
                      { parser : org.apache.tika.parser.jpeg.JpegParser }
                      { parser : org.apache.tika.parser.mail.RFC822Parser }
                      { parser : org.apache.tika.parser.mbox.MboxParser,
                          additionalSupportedMimeTypes : [message/x-emlx] }
                      { parser : org.apache.tika.parser.microsoft.OfficeParser }
                      { parser : org.apache.tika.parser.microsoft.TNEFParser }
                      { parser : org.apache.tika.parser.microsoft.ooxml.OOXMLParser }
                      { parser : org.apache.tika.parser.mp3.Mp3Parser }
                      { parser : org.apache.tika.parser.mp4.MP4Parser }
                      { parser : org.apache.tika.parser.hdf.HDFParser }
                      { parser : org.apache.tika.parser.netcdf.NetCDFParser }
                      { parser : org.apache.tika.parser.odf.OpenDocumentParser }
                      { parser : org.apache.tika.parser.pdf.PDFParser }
                      { parser : org.apache.tika.parser.pkg.CompressorParser }
                      { parser : org.apache.tika.parser.pkg.PackageParser }
                      { parser : org.apache.tika.parser.rtf.RTFParser }
                      { parser : org.apache.tika.parser.txt.TXTParser }
                      { parser : org.apache.tika.parser.video.FLVParser }
                      { parser : org.apache.tika.parser.xml.DcXMLParser }
                      { parser : org.apache.tika.parser.xml.FictionBookParser }
                      { parser : org.apache.tika.parser.chm.ChmParser }
                    ]
                  }
                }

                { generateUUID { field : ignored_base_id } }

                {
                  generateSolrSequenceKey {
                    baseIdField: ignored_base_id
                    solrLocator : ${SOLR_LOCATOR}
                  }
                }

              ]
            }
          ]
        }
      }

      {
        loadSolr {
          solrLocator : ${SOLR_LOCATOR}
        }
      }

      {
        logDebug {
          format : "My output record: {}"
          args : ["@{}"]
        }
      }

    ]
  }
]

userAgent {
  inputField : user_agents
  outputFields : {
    ua_family : "@{ua_family}"
    device_family : "@{device_family}"
    ua_family_and_version : "@{ua_family}/@{ua_major}.@{ua_minor}.@{ua_patch}"
    os_family_and_version : "@{os_family} @{os_major}.@{os_minor}.@{os_patch}"
  }          
}

user_agents : Mozilla/5.0 (iPhone; CPU iPhone OS 5_1_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko) Version/5.1 Mobile/9B206 Safari/7534.48.3

ua_family : Mobile Safari
device_family : iPhone
ua_family_and_version : Mobile Safari/5.1
os_family_and_version : iOS 5.1.1