View Javadoc

    1   /*
    2    * Licensed to the Apache Software Foundation (ASF) under one or more
    3    * contributor license agreements.  See the NOTICE file distributed with
    4    * this work for additional information regarding copyright ownership.
    5    * The ASF licenses this file to You under the Apache License, Version 2.0
    6    * (the "License"); you may not use this file except in compliance with
    7    * the License.  You may obtain a copy of the License at
    8    *
    9    *      http://www.apache.org/licenses/LICENSE-2.0
    10   *
    11   * Unless required by applicable law or agreed to in writing, software
    12   * distributed under the License is distributed on an "AS IS" BASIS,
    13   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    14   * See the License for the specific language governing permissions and
    15   * limitations under the License.
    16   */
    17
    18  package org.apache.log4j.varia;
    19
    20  import java.io.BufferedReader;
    21  import java.io.FileNotFoundException;
    22  import java.io.IOException;
    23  import java.io.InputStreamReader;
    24  import java.io.Reader;
    25  import java.net.MalformedURLException;
    26  import java.net.URL;
    27  import java.text.SimpleDateFormat;
    28  import java.util.ArrayList;
    29  import java.util.HashMap;
    30  import java.util.Hashtable;
    31  import java.util.Iterator;
    32  import java.util.List;
    33  import java.util.Map;
    34  import java.util.StringTokenizer;
    35  import java.util.regex.MatchResult;
    36  import java.util.regex.Matcher;
    37  import java.util.regex.Pattern;
    38  import java.util.regex.PatternSyntaxException;
    39
    40  import org.apache.log4j.Level;
    41  import org.apache.log4j.Logger;
    42  import org.apache.log4j.helpers.Constants;
    43  import org.apache.log4j.plugins.Receiver;
    44  import org.apache.log4j.rule.ExpressionRule;
    45  import org.apache.log4j.rule.Rule;
    46  import org.apache.log4j.spi.LocationInfo;
    47  import org.apache.log4j.spi.LoggingEvent;
    48  import org.apache.log4j.spi.ThrowableInformation;
    49
    50  /**
    51   * LogFilePatternReceiver can parse and tail log files, converting entries into
    52   * LoggingEvents.  If the file doesn't exist when the receiver is initialized, the
    53   * receiver will look for the file once every 10 seconds.
    54   * <p>
    55   * This receiver relies on java.util.regex features to perform the parsing of text in the
    56   * log file, however the only regular expression field explicitly supported is
    57   * a glob-style wildcard used to ignore fields in the log file if needed.  All other
    58   * fields are parsed by using the supplied keywords.
    59   * <p>
    60   * <b>Features:</b><br>
    61   * - specify the URL of the log file to be processed<br>
    62   * - specify the timestamp format in the file (if one exists, using patterns from {@link java.text.SimpleDateFormat})<br>
    63   * - specify the pattern (logFormat) used in the log file using keywords, a wildcard character (*) and fixed text<br>
    64   * - 'tail' the file (allows the contents of the file to be continually read and new events processed)<br>
    65   * - supports the parsing of multi-line messages and exceptions
    66   * - 'hostname' property set to URL host (or 'file' if not available)
    67   * - 'application' property set to URL path (or value of fileURL if not available)
    68   * - 'group' property can be set to associate multiple log file receivers
    69   *<p>
    70   * <b>Keywords:</b><br>
    71   * TIMESTAMP<br>
    72   * LOGGER<br>
    73   * LEVEL<br>
    74   * THREAD<br>
    75   * CLASS<br>
    76   * FILE<br>
    77   * LINE<br>
    78   * METHOD<br>
    79   * RELATIVETIME<br>
    80   * MESSAGE<br>
    81   * NDC<br>
    82   * PROP(key)<br>
    83   * (NL)<br>
    84   * <p>
    85   * (NL) represents a new line embedded in the log format, supporting log formats whose fields span multiple lines
    86   * <p>
    87   * Use a * to ignore portions of the log format that should be ignored
    88   * <p>
    89   * Example:<br>
    90   * If your file's patternlayout is this:<br>
    91   * <b>%d %-5p [%t] %C{2} (%F:%L) - %m%n</b>
    92   *<p>
    93   * specify this as the log format:<br>
    94   * <b>TIMESTAMP LEVEL [THREAD] CLASS (FILE:LINE) - MESSAGE</b>
    95   *<p>
    96   * To define a PROPERTY field, use PROP(key)
    97   * <p>
    98   * Example:<br>
    99   * If you used the RELATIVETIME pattern layout character in the file,
    100  * you can use PROP(RELATIVETIME) in the logFormat definition to assign
    101  * the RELATIVETIME field as a property on the event.
    102  * <p>
    103  * If your file's patternlayout is this:<br>
    104  * <b>%r [%t] %-5p %c %x - %m%n</b>
    105  *<p>
    106  * specify this as the log format:<br>
    107  * <b>PROP(RELATIVETIME) [THREAD] LEVEL LOGGER * - MESSAGE</b>
    108  * <p>
    109  * Note the * - it can be used to ignore a single word or sequence of words in the log file
    110  * (in order for the wildcard to ignore a sequence of words, the text being ignored must be
    111  *  followed by some delimiter, like '-' or '[') - ndc is being ignored in the following example.
    112  * <p>
    113  * Assign a filterExpression in order to only process events which match a filter.
    114  * If a filterExpression is not assigned, all events are processed.
    115  *<p>
    116  * <b>Limitations:</b><br>
    117  * - no support for the single-line version of throwable supported by patternlayout<br>
    118  *   (this version of throwable will be included as the last line of the message)<br>
    119  * - the relativetime patternLayout character must be set as a property: PROP(RELATIVETIME)<br>
    120  * - messages should appear as the last field of the logFormat because the variability in message content<br>
    121  * - exceptions are converted if the exception stack trace (other than the first line of the exception)<br>
    122  *   is stored in the log file with a tab followed by the word 'at' as the first characters in the line<br>
    123  * - tailing may fail if the file rolls over.
    124  *<p>
    125  * <b>Example receiver configuration settings</b> (add these as params, specifying a LogFilePatternReceiver 'plugin'):<br>
    126  * param: "timestampFormat" value="yyyy-MM-d HH:mm:ss,SSS"<br>
    127  * param: "logFormat" value="PROP(RELATIVETIME) [THREAD] LEVEL LOGGER * - MESSAGE"<br>
    128  * param: "fileURL" value="file:///c:/events.log"<br>
    129  * param: "tailing" value="true"
    130  *<p>
    131  * This configuration will be able to process these sample events:<br>
    132  * 710    [       Thread-0] DEBUG                   first.logger first - &lt;test&gt;   &lt;test2&gt;something here&lt;/test2&gt;   &lt;test3 blah=something/&gt;   &lt;test4&gt;       &lt;test5&gt;something else&lt;/test5&gt;   &lt;/test4&gt;&lt;/test&gt;<br>
    133  * 880    [       Thread-2] DEBUG                   first.logger third - &lt;test&gt;   &lt;test2&gt;something here&lt;/test2&gt;   &lt;test3 blah=something/&gt;   &lt;test4&gt;       &lt;test5&gt;something else&lt;/test5&gt;   &lt;/test4&gt;&lt;/test&gt;<br>
    134  * 880    [       Thread-0] INFO                    first.logger first - infomsg-0<br>
    135  * java.lang.Exception: someexception-first<br>
    136  *     at Generator2.run(Generator2.java:102)<br>
    137  *
    138  *@author Scott Deboy
    139  */
    140 public class LogFilePatternReceiver extends Receiver {
    141   private final List keywords = new ArrayList();
    142
    143   private static final String PROP_START = "PROP(";
    144   private static final String PROP_END = ")";
    145
    146   private static final String LOGGER = "LOGGER";
    147   private static final String MESSAGE = "MESSAGE";
    148   private static final String TIMESTAMP = "TIMESTAMP";
    149   private static final String NDC = "NDC";
    150   private static final String LEVEL = "LEVEL";
    151   private static final String THREAD = "THREAD";
    152   private static final String CLASS = "CLASS";
    153   private static final String FILE = "FILE";
    154   private static final String LINE = "LINE";
    155   private static final String METHOD = "METHOD";
    156   private static final String NEWLINE = "(NL)";
    157
    158   private static final String DEFAULT_HOST = "file";
    159
    160   //all lines other than first line of exception begin with tab followed by 'at' followed by text
    161   private static final String EXCEPTION_PATTERN = "^\\s+at.*";
    162   private static final String REGEXP_DEFAULT_WILDCARD = ".*?";
    163   private static final String REGEXP_GREEDY_WILDCARD = ".*";
    164   private static final String PATTERN_WILDCARD = "*";
    165   //pull in optional leading and trailing spaces
    166   private static final String NOSPACE_GROUP = "(\\s*?\\S*?\\s*?)";
    167   private static final String DEFAULT_GROUP = "(" + REGEXP_DEFAULT_WILDCARD + ")";
    168   private static final String GREEDY_GROUP = "(" + REGEXP_GREEDY_WILDCARD + ")";
    169   private static final String MULTIPLE_SPACES_REGEXP = "[ ]+";
    170   private static final String NEWLINE_REGEXP = "\n";
    171   private final String newLine = System.getProperty("line.separator");
    172
    173   private final String[] emptyException = new String[] { "" };
    174
    175   private SimpleDateFormat dateFormat;
    176   private String timestampFormat;
    177   private String logFormat;
    178   private String customLevelDefinitions;
    179   private String fileURL;
    180   private String host;
    181   private String path;
    182   private boolean tailing;
    183   private String filterExpression;
    184   private long waitMillis = 2000; //default 2 seconds
    185   private String group;
    186
    187   private static final String VALID_DATEFORMAT_CHARS = "GyYMwWDdFEuaHkKhmsSzZX";
    188   private static final String VALID_DATEFORMAT_CHAR_PATTERN = "[" + VALID_DATEFORMAT_CHARS + "]";
    189
    190   private Rule expressionRule;
    191
    192   private Map currentMap;
    193   private List additionalLines;
    194   private List matchingKeywords;
    195
    196   private String regexp;
    197   private Reader reader;
    198   private Pattern regexpPattern;
    199   private Pattern exceptionPattern;
    200   private String timestampPatternText;
    201
    202   private boolean useCurrentThread;
    203   public static final int MISSING_FILE_RETRY_MILLIS = 10000;
    204   private boolean appendNonMatches;
    205   private final Map customLevelDefinitionMap = new HashMap();
    206
    207   //default to one line - this number is incremented for each (NL) found in the logFormat
    208   private int lineCount = 1;
    209
    210     public LogFilePatternReceiver() {
    211     keywords.add(TIMESTAMP);
    212     keywords.add(LOGGER);
    213     keywords.add(LEVEL);
    214     keywords.add(THREAD);
    215     keywords.add(CLASS);
    216     keywords.add(FILE);
    217     keywords.add(LINE);
    218     keywords.add(METHOD);
    219     keywords.add(MESSAGE);
    220     keywords.add(NDC);
    221     try {
    222         exceptionPattern = Pattern.compile(EXCEPTION_PATTERN);
    223     } catch (PatternSyntaxException pse) {
    224         //shouldn't happen
    225     }
    226   }
    227
    228   /**
    229    * Accessor
    230    *
    231    * @return file URL
    232    */
    233   public String getFileURL() {
    234     return fileURL;
    235   }
    236
    237   /**
    238    * Mutator
    239    *
    240    * @param fileURL
    241    */
    242   public void setFileURL(String fileURL) {
    243     this.fileURL = fileURL;
    244   }
    245
    246     /**
    247      * If the log file contains non-log4j level strings, they can be mapped to log4j levels using the format (android example):
    248      * V=TRACE,D=DEBUG,I=INFO,W=WARN,E=ERROR,F=FATAL,S=OFF
    249      *
    250      * @param customLevelDefinitions the level definition string
    251      */
    252   public void setCustomLevelDefinitions(String customLevelDefinitions) {
    253     this.customLevelDefinitions = customLevelDefinitions;
    254   }
    255
    256   public String getCustomLevelDefinitions() {
    257     return customLevelDefinitions;
    258   }
    259
    260   /**
    261    * Accessor
    262    * @return append non matches
    263    */
    264   public boolean isAppendNonMatches() {
    265       return appendNonMatches;
    266   }
    267
    268   /**
    269    * Mutator
    270    * @param appendNonMatches
    271    */
    272   public void setAppendNonMatches(boolean appendNonMatches) {
    273       this.appendNonMatches = appendNonMatches;
    274   }
    275
    276   /**
    277    * Accessor
    278    *
    279    * @return filter expression
    280    */
    281   public String getFilterExpression() {
    282     return filterExpression;
    283   }
    284
    285   /**
    286    * Mutator
    287    *
    288    * @param filterExpression
    289    */
    290   public void setFilterExpression(String filterExpression) {
    291     this.filterExpression = filterExpression;
    292   }
    293
    294   /**
    295    * Accessor
    296    *
    297    * @return tailing
    298    */
    299   public boolean isTailing() {
    300     return tailing;
    301   }
    302
    303   /**
    304    * Mutator
    305    *
    306    * @param tailing
    307    */
    308   public void setTailing(boolean tailing) {
    309     this.tailing = tailing;
    310   }
    311
    312   /**
    313    * When true, this property uses the current Thread to perform the import,
    314    * otherwise when false (the default), a new Thread is created and started to manage
    315    * the import.
    316    * @return
    317    */
    318  public final boolean isUseCurrentThread() {
    319      return useCurrentThread;
    320  }
    321
    322  /**
    323   * Sets whether the current Thread or a new Thread is created to perform the import,
    324   * the default being false (new Thread created).
    325   *
    326   * @param useCurrentThread
    327   */
    328  public final void setUseCurrentThread(boolean useCurrentThread) {
    329      this.useCurrentThread = useCurrentThread;
    330  }
    331
    332   /**
    333    * Accessor
    334    *
    335    * @return log format
    336    */
    337   public String getLogFormat() {
    338     return logFormat;
    339   }
    340
    341     /**
    342    * Mutator
    343    *
    344    * @param logFormat
    345    *          the format
    346    */
    347   public void setLogFormat(String logFormat) {
    348     this.logFormat = logFormat;
    349   }
    350
    351     /**
    352    * Mutator
    353    */
    354   public void setGroup(String group) { this.group = group; }
    355
    356
    357     /**
    358    * Accessor
    359    *
    360    * @return group
    361    */
    362   public String getGroup() { return group; }
    363
    364     /**
    365    * Mutator.  Specify a pattern from {@link java.text.SimpleDateFormat}
    366    *
    367    * @param timestampFormat
    368    */
    369   public void setTimestampFormat(String timestampFormat) {
    370     this.timestampFormat = timestampFormat;
    371   }
    372
    373     /**
    374    * Accessor
    375    *
    376    * @return timestamp format
    377    */
    378   public String getTimestampFormat() {
    379     return timestampFormat;
    380   }
    381
    382   /**
    383    * Accessor
    384    * @return millis between retrieves of content
    385    */
    386   public long getWaitMillis() {
    387     return waitMillis;
    388   }
    389
    390   /**
    391    * Mutator
    392    * @param waitMillis
    393    */
    394   public void setWaitMillis(long waitMillis) {
    395     this.waitMillis = waitMillis;
    396   }
    397
    398     /**
    399    * Walk the additionalLines list, looking for the EXCEPTION_PATTERN.
    400    * <p>
    401    * Return the index of the first matched line
    402    * (the match may be the 1st line of an exception)
    403    * <p>
    404    * Assumptions: <br>
    405    * - the additionalLines list may contain both message and exception lines<br>
    406    * - message lines are added to the additionalLines list and then
    407    * exception lines (all message lines occur in the list prior to all
    408    * exception lines)
    409    *
    410    * @return -1 if no exception line exists, line number otherwise
    411    */
    412   private int getExceptionLine() {
    413     for (int i = 0; i < additionalLines.size(); i++) {
    414       Matcher exceptionMatcher = exceptionPattern.matcher((String)additionalLines.get(i));
    415       if (exceptionMatcher.matches()) {
    416         return i;
    417       }
    418     }
    419     return -1;
    420   }
    421
    422     /**
    423    * Combine all message lines occuring in the additionalLines list, adding
    424    * a newline character between each line
    425    * <p>
    426    * the event will already have a message - combine this message
    427    * with the message lines in the additionalLines list
    428    * (all entries prior to the exceptionLine index)
    429    *
    430    * @param firstMessageLine primary message line
    431    * @param exceptionLine index of first exception line
    432    * @return message
    433    */
    434   private String buildMessage(String firstMessageLine, int exceptionLine) {
    435     if (additionalLines.size() == 0) {
    436       return firstMessageLine;
    437     }
    438     StringBuffer message = new StringBuffer();
    439     if (firstMessageLine != null) {
    440       message.append(firstMessageLine);
    441     }
    442
    443     int linesToProcess = (exceptionLine == -1?additionalLines.size(): exceptionLine);
    444
    445     for (int i = 0; i < linesToProcess; i++) {
    446       message.append(newLine);
    447       message.append(additionalLines.get(i));
    448     }
    449     return message.toString();
    450   }
    451
    452     /**
    453    * Combine all exception lines occuring in the additionalLines list into a
    454    * String array
    455    * <p>
    456    * (all entries equal to or greater than the exceptionLine index)
    457    *
    458    * @param exceptionLine index of first exception line
    459    * @return exception
    460    */
    461   private String[] buildException(int exceptionLine) {
    462     if (exceptionLine == -1) {
    463       return emptyException;
    464     }
    465     String[] exception = new String[additionalLines.size() - exceptionLine - 1];
    466     for (int i = 0; i < exception.length; i++) {
    467       exception[i] = (String) additionalLines.get(i + exceptionLine);
    468     }
    469     return exception;
    470   }
    471
    472     /**
    473    * Construct a logging event from currentMap and additionalLines
    474    * (additionalLines contains multiple message lines and any exception lines)
    475    * <p>
    476    * CurrentMap and additionalLines are cleared in the process
    477    *
    478    * @return event
    479    */
    480   private LoggingEvent buildEvent() {
    481     if (currentMap.size() == 0) {
    482       if (additionalLines.size() > 0) {
    483         for (Iterator iter = additionalLines.iterator();iter.hasNext();) {
    484           getLogger().debug("found non-matching line: " + iter.next());
    485         }
    486       }
    487       additionalLines.clear();
    488       return null;
    489     }
    490     //the current map contains fields - build an event
    491     int exceptionLine = getExceptionLine();
    492     String[] exception = buildException(exceptionLine);
    493
    494     //messages are listed before exceptions in additionallines
    495     if (additionalLines.size() > 0 && exception.length > 0) {
    496       currentMap.put(MESSAGE, buildMessage((String) currentMap.get(MESSAGE),
    497           exceptionLine));
    498     }
    499     LoggingEvent event = convertToEvent(currentMap, exception);
    500     currentMap.clear();
    501     additionalLines.clear();
    502     return event;
    503   }
    504
    505     /**
    506    * Read, parse and optionally tail the log file, converting entries into logging events.
    507    *
    508    * A runtimeException is thrown if the logFormat pattern is malformed.
    509    *
    510    * @param bufferedReader
    511    * @throws IOException
    512    */
    513   protected void process(BufferedReader bufferedReader) throws IOException {
    514         Matcher eventMatcher;
    515         Matcher exceptionMatcher;
    516         String line;
    517         //if newlines are provided in the logFormat - (NL) - combine the lines prior to matching
    518         while ((line = bufferedReader.readLine()) != null) {
    519             //there is already one line (read above, start i at 1
    520             for (int i=1;i<lineCount;i++)
    521             {
    522                 String thisLine = bufferedReader.readLine();
    523                 if (thisLine != null)
    524                 {
    525                   line = line + newLine + thisLine;
    526                 }
    527             }
    528             eventMatcher = regexpPattern.matcher(line);
    529             //skip empty line entries
    530             if (line.trim().equals("")) {continue;}
    531             exceptionMatcher = exceptionPattern.matcher(line);
    532             if (eventMatcher.matches()) {
    533                 //build an event from the previous match (held in current map)
    534                 LoggingEvent event = buildEvent();
    535                 if (event != null) {
    536                     if (passesExpression(event)) {
    537                         doPost(event);
    538                     }
    539                 }
    540                 currentMap.putAll(processEvent(eventMatcher.toMatchResult()));
    541             } else if (exceptionMatcher.matches()) {
    542                 //an exception line
    543                 additionalLines.add(line);
    544             } else {
    545                 //neither...either post an event with the line or append as additional lines
    546                 //if this was a logging event with multiple lines, each line will show up as its own event instead of being
    547                 //appended as multiple lines on the same event..
    548                 //choice is to have each non-matching line show up as its own line, or append them all to a previous event
    549                 if (appendNonMatches) {
    550                     //hold on to the previous time, so we can do our best to preserve time-based ordering if the event is a non-match
    551                     String lastTime = (String)currentMap.get(TIMESTAMP);
    552                     //build an event from the previous match (held in current map)
    553                     if (currentMap.size() > 0) {
    554                         LoggingEvent event = buildEvent();
    555                         if (event != null) {
    556                             if (passesExpression(event)) {
    557                               doPost(event);
    558                             }
    559                         }
    560                     }
    561                     if (lastTime != null) {
    562                         currentMap.put(TIMESTAMP, lastTime);
    563                     }
    564                     currentMap.put(MESSAGE, line);
    565                 } else {
    566                     additionalLines.add(line);
    567                 }
    568             }
    569         }
    570
    571         //process last event if one exists
    572         LoggingEvent event = buildEvent();
    573         if (event != null) {
    574             if (passesExpression(event)) {
    575                 doPost(event);
    576             }
    577         }
    578     }
    579
    580     protected void createPattern() {
    581         regexpPattern = Pattern.compile(regexp);
    582     }
    583
    584     /**
    585    * Helper method that supports the evaluation of the expression
    586    *
    587    * @param event
    588    * @return true if expression isn't set, or the result of the evaluation otherwise
    589    */
    590   private boolean passesExpression(LoggingEvent event) {
    591     if (event != null) {
    592       if (expressionRule != null) {
    593         return (expressionRule.evaluate(event, null));
    594       }
    595     }
    596     return true;
    597   }
    598
    599     /**
    600    * Convert the match into a map.
    601    * <p>
    602    * Relies on the fact that the matchingKeywords list is in the same
    603    * order as the groups in the regular expression
    604    *
    605    * @param result
    606    * @return map
    607    */
    608   private Map processEvent(MatchResult result) {
    609     Map map = new HashMap();
    610     //group zero is the entire match - process all other groups
    611     for (int i = 1; i < result.groupCount() + 1; i++) {
    612       Object key = matchingKeywords.get(i - 1);
    613       Object value = result.group(i);
    614       map.put(key, value);
    615
    616     }
    617     return map;
    618   }
    619
    620     /**
    621    * Helper method that will convert timestamp format to a pattern
    622    *
    623    *
    624    * @return string
    625    */
    626   private String convertTimestamp() {
    627     //some locales (for example, French) generate timestamp text with characters not included in \w -
    628     // now using \S (all non-whitespace characters) instead of /w
    629     String result = "";
    630     if (timestampFormat != null) {
    631       result = timestampFormat.replaceAll(Pattern.quote("+"), "[+]");
    632       result = result.replaceAll(VALID_DATEFORMAT_CHAR_PATTERN, "\\\\S+");
    633       //make sure dots in timestamp are escaped
    634       result = result.replaceAll(Pattern.quote("."), "\\\\.");
    635     }
    636     return result;
    637   }
    638
    639     protected void setHost(String host) {
    640 	  this.host = host;
    641   }
    642
    643     protected void setPath(String path) {
    644 	  this.path = path;
    645   }
    646
    647   public String getPath() {
    648       return path;
    649   }
    650
    651     /**
    652    * Build the regular expression needed to parse log entries
    653    *
    654    */
    655   protected void initialize() {
    656 	if (host == null && path == null) {
    657 		try {
    658 			URL url = new URL(fileURL);
    659 			host = url.getHost();
    660 			path = url.getPath();
    661 		} catch (MalformedURLException e1) {
    662 			// TODO Auto-generated catch block
    663 			e1.printStackTrace();
    664 		}
    665 	}
    666 	if (host == null || host.trim().equals("")) {
    667 		host = DEFAULT_HOST;
    668 	}
    669 	if (path == null || path.trim().equals("")) {
    670 		path = fileURL;
    671 	}
    672
    673     currentMap = new HashMap();
    674     additionalLines = new ArrayList();
    675     matchingKeywords = new ArrayList();
    676
    677     if (timestampFormat != null) {
    678       dateFormat = new SimpleDateFormat(quoteTimeStampChars(timestampFormat));
    679       timestampPatternText = convertTimestamp();
    680     }
    681     //if custom level definitions exist, parse them
    682     updateCustomLevelDefinitionMap();
    683     try {
    684       if (filterExpression != null) {
    685         expressionRule = ExpressionRule.getRule(filterExpression);
    686       }
    687     } catch (Exception e) {
    688       getLogger().warn("Invalid filter expression: " + filterExpression, e);
    689     }
    690
    691     List buildingKeywords = new ArrayList();
    692
    693     String newPattern = logFormat;
    694
    695     //process newlines - (NL) - in the logFormat - before processing properties
    696     int index = 0;
    697     while (index > -1) {
    698       index = newPattern.indexOf(NEWLINE);
    699       if (index > -1) {
    700         //keep track of number of expected newlines in the format, so the lines can be concatenated prior to matching
    701         lineCount++;
    702         newPattern = singleReplace(newPattern, NEWLINE, NEWLINE_REGEXP);
    703       }
    704     }
    705
    706     String current = newPattern;
    707     //build a list of property names and temporarily replace the property with an empty string,
    708     //we'll rebuild the pattern later
    709     List propertyNames = new ArrayList();
    710     index = 0;
    711     while (index > -1) {
    712         if (current.indexOf(PROP_START) > -1 && current.indexOf(PROP_END) > -1) {
    713             index = current.indexOf(PROP_START);
    714             String longPropertyName = current.substring(current.indexOf(PROP_START), current.indexOf(PROP_END) + 1);
    715             String shortProp = getShortPropertyName(longPropertyName);
    716             buildingKeywords.add(shortProp);
    717             propertyNames.add(longPropertyName);
    718             current = current.substring(longPropertyName.length() + 1 + index);
    719             newPattern = singleReplace(newPattern, longPropertyName, new Integer(buildingKeywords.size() -1).toString());
    720         } else {
    721             //no properties
    722             index = -1;
    723         }
    724     }
    725
    726     /*
    727      * we're using a treemap, so the index will be used as the key to ensure
    728      * keywords are ordered correctly
    729      *
    730      * examine pattern, adding keywords to an index-based map patterns can
    731      * contain only one of these per entry...properties are the only 'keyword'
    732      * that can occur multiple times in an entry
    733      */
    734     Iterator iter = keywords.iterator();
    735     while (iter.hasNext()) {
    736       String keyword = (String) iter.next();
    737       int index2 = newPattern.indexOf(keyword);
    738       if (index2 > -1) {
    739         buildingKeywords.add(keyword);
    740         newPattern = singleReplace(newPattern, keyword, new Integer(buildingKeywords.size() -1).toString());
    741       }
    742     }
    743
    744     String buildingInt = "";
    745
    746     for (int i=0;i<newPattern.length();i++) {
    747         String thisValue = String.valueOf(newPattern.substring(i, i+1));
    748         if (isInteger(thisValue)) {
    749             buildingInt = buildingInt + thisValue;
    750         } else {
    751             if (isInteger(buildingInt)) {
    752                 matchingKeywords.add(buildingKeywords.get(Integer.parseInt(buildingInt)));
    753             }
    754             //reset
    755             buildingInt = "";
    756         }
    757     }
    758
    759     //if the very last value is an int, make sure to add it
    760     if (isInteger(buildingInt)) {
    761         matchingKeywords.add(buildingKeywords.get(Integer.parseInt(buildingInt)));
    762     }
    763
    764     newPattern = replaceMetaChars(newPattern);
    765
    766     //compress one or more spaces in the pattern into the [ ]+ regexp
    767     //(supports padding of level in log files)
    768     newPattern = newPattern.replaceAll(MULTIPLE_SPACES_REGEXP, MULTIPLE_SPACES_REGEXP);
    769     newPattern = newPattern.replaceAll(Pattern.quote(PATTERN_WILDCARD), REGEXP_DEFAULT_WILDCARD);
    770     //use buildingKeywords here to ensure correct order
    771     for (int i = 0;i<buildingKeywords.size();i++) {
    772       String keyword = (String) buildingKeywords.get(i);
    773       //make the final keyword greedy (we're assuming it's the message)
    774       if (i == (buildingKeywords.size() - 1)) {
    775         newPattern = singleReplace(newPattern, String.valueOf(i), GREEDY_GROUP);
    776       } else if (TIMESTAMP.equals(keyword)) {
    777         newPattern = singleReplace(newPattern, String.valueOf(i), "(" + timestampPatternText + ")");
    778       } else if (LOGGER.equals(keyword) || LEVEL.equals(keyword)) {
    779         newPattern = singleReplace(newPattern, String.valueOf(i), NOSPACE_GROUP);
    780       } else {
    781         newPattern = singleReplace(newPattern, String.valueOf(i), DEFAULT_GROUP);
    782       }
    783     }
    784
    785     regexp = newPattern;
    786     getLogger().debug("regexp is " + regexp);
    787   }
    788
    789     private void updateCustomLevelDefinitionMap() {
    790         if (customLevelDefinitions != null) {
    791             StringTokenizer entryTokenizer = new StringTokenizer(customLevelDefinitions, ",");
    792
    793             customLevelDefinitionMap.clear();
    794             while (entryTokenizer.hasMoreTokens()) {
    795                 StringTokenizer innerTokenizer = new StringTokenizer(entryTokenizer.nextToken(), "=");
    796                 customLevelDefinitionMap.put(innerTokenizer.nextToken(), Level.toLevel(innerTokenizer.nextToken()));
    797             }
    798         }
    799     }
    800
    801     private boolean isInteger(String value) {
    802         try {
    803             Integer.parseInt(value);
    804             return true;
    805         } catch (NumberFormatException nfe) {
    806             return false;
    807         }
    808     }
    809
    810     private String quoteTimeStampChars(String input) {
    811         //put single quotes around text that isn't a supported dateformat char
    812         StringBuffer result = new StringBuffer();
    813         //ok to default to false because we also check for index zero below
    814         boolean lastCharIsDateFormat = false;
    815         for (int i = 0;i<input.length();i++) {
    816             String thisVal = input.substring(i, i + 1);
    817             boolean thisCharIsDateFormat = VALID_DATEFORMAT_CHARS.contains(thisVal);
    818             //we have encountered a non-dateformat char
    819             if (!thisCharIsDateFormat && (i == 0 || lastCharIsDateFormat)) {
    820                 result.append("'");
    821             }
    822             //we have encountered a dateformat char after previously encountering a non-dateformat char
    823             if (thisCharIsDateFormat && i > 0 && !lastCharIsDateFormat) {
    824                 result.append("'");
    825             }
    826             lastCharIsDateFormat = thisCharIsDateFormat;
    827             result.append(thisVal);
    828         }
    829         //append an end single-quote if we ended with non-dateformat char
    830         if (!lastCharIsDateFormat) {
    831             result.append("'");
    832         }
    833         return result.toString();
    834     }
    835
    836     private String singleReplace(String inputString, String oldString, String newString)
    837     {
    838         int propLength = oldString.length();
    839         int startPos = inputString.indexOf(oldString);
    840         if (startPos == -1)
    841         {
    842             getLogger().info("string: " + oldString + " not found in input: " + inputString + " - returning input");
    843             return inputString;
    844         }
    845         if (startPos == 0)
    846         {
    847             inputString = inputString.substring(propLength);
    848             inputString = newString + inputString;
    849         } else {
    850             inputString = inputString.substring(0, startPos) + newString + inputString.substring(startPos + propLength);
    851         }
    852         return inputString;
    853     }
    854
    855     private String getShortPropertyName(String longPropertyName)
    856   {
    857       String currentProp = longPropertyName.substring(longPropertyName.indexOf(PROP_START));
    858       String prop = currentProp.substring(0, currentProp.indexOf(PROP_END) + 1);
    859       String shortProp = prop.substring(PROP_START.length(), prop.length() - 1);
    860       return shortProp;
    861   }
    862
    863     /**
    864    * Some perl5 characters may occur in the log file format.
    865    * Escape these characters to prevent parsing errors.
    866    *
    867    * @param input
    868    * @return string
    869    */
    870   private String replaceMetaChars(String input) {
    871     //escape backslash first since that character is used to escape the remaining meta chars
    872     input = input.replaceAll("\\\\", "\\\\\\");
    873
    874     //don't escape star - it's used as the wildcard
    875     input = input.replaceAll(Pattern.quote("]"), "\\\\]");
    876     input = input.replaceAll(Pattern.quote("["), "\\\\[");
    877     input = input.replaceAll(Pattern.quote("^"), "\\\\^");
    878     input = input.replaceAll(Pattern.quote("$"), "\\\\$");
    879     input = input.replaceAll(Pattern.quote("."), "\\\\.");
    880     input = input.replaceAll(Pattern.quote("|"), "\\\\|");
    881     input = input.replaceAll(Pattern.quote("?"), "\\\\?");
    882     input = input.replaceAll(Pattern.quote("+"), "\\\\+");
    883     input = input.replaceAll(Pattern.quote("("), "\\\\(");
    884     input = input.replaceAll(Pattern.quote(")"), "\\\\)");
    885     input = input.replaceAll(Pattern.quote("-"), "\\\\-");
    886     input = input.replaceAll(Pattern.quote("{"), "\\\\{");
    887     input = input.replaceAll(Pattern.quote("}"), "\\\\}");
    888     input = input.replaceAll(Pattern.quote("#"), "\\\\#");
    889     return input;
    890   }
    891
    892     /**
    893    * Convert a keyword-to-values map to a LoggingEvent
    894    *
    895    * @param fieldMap
    896    * @param exception
    897    *
    898    * @return logging event
    899    */
    900   private LoggingEvent convertToEvent(Map fieldMap, String[] exception) {
    901     if (fieldMap == null) {
    902       return null;
    903     }
    904
    905     //a logger must exist at a minimum for the event to be processed
    906     if (!fieldMap.containsKey(LOGGER)) {
    907       fieldMap.put(LOGGER, "Unknown");
    908     }
    909     if (exception == null) {
    910       exception = emptyException;
    911     }
    912
    913     Logger logger = null;
    914     long timeStamp = 0L;
    915     String level = null;
    916     String threadName = null;
    917     Object message = null;
    918     String ndc = null;
    919     String className = null;
    920     String methodName = null;
    921     String eventFileName = null;
    922     String lineNumber = null;
    923     Hashtable properties = new Hashtable();
    924
    925     logger = Logger.getLogger((String) fieldMap.remove(LOGGER));
    926
    927     if ((dateFormat != null) && fieldMap.containsKey(TIMESTAMP)) {
    928       try {
    929         timeStamp = dateFormat.parse((String) fieldMap.remove(TIMESTAMP))
    930             .getTime();
    931       } catch (Exception e) {
    932         e.printStackTrace();
    933       }
    934     }
    935     //use current time if timestamp not parseable/dateformat not specified
    936     if (timeStamp == 0L) {
    937       timeStamp = System.currentTimeMillis();
    938     }
    939
    940     message = fieldMap.remove(MESSAGE);
    941     if (message == null) {
    942       message = "";
    943     }
    944
    945     level = (String) fieldMap.remove(LEVEL);
    946     Level levelImpl;
    947     if (level == null) {
    948         levelImpl = Level.DEBUG;
    949     } else {
    950         //first try to resolve against custom level definition map, then fall back to regular levels
    951         levelImpl = (Level) customLevelDefinitionMap.get(level);
    952         if (levelImpl == null) {
    953             levelImpl = Level.toLevel(level.trim());
    954             if (!level.equals(levelImpl.toString())) {
    955                 //check custom level map
    956                 if (levelImpl == null) {
    957                     levelImpl = Level.DEBUG;
    958                     getLogger().debug("found unexpected level: " + level + ", logger: " + logger.getName() + ", msg: " + message);
    959                     //make sure the text that couldn't match a level is added to the message
    960                     message = level + " " + message;
    961                 }
    962             }
    963         }
    964     }
    965
    966     threadName = (String) fieldMap.remove(THREAD);
    967
    968     ndc = (String) fieldMap.remove(NDC);
    969
    970     className = (String) fieldMap.remove(CLASS);
    971
    972     methodName = (String) fieldMap.remove(METHOD);
    973
    974     eventFileName = (String) fieldMap.remove(FILE);
    975
    976     lineNumber = (String) fieldMap.remove(LINE);
    977
    978     properties.put(Constants.HOSTNAME_KEY, host);
    979     properties.put(Constants.APPLICATION_KEY, path);
    980     properties.put(Constants.RECEIVER_NAME_KEY, getName());
    981     if (group != null) {
    982         properties.put(Constants.GROUP_KEY, group);
    983     }
    984
    985     //all remaining entries in fieldmap are properties
    986     properties.putAll(fieldMap);
    987
    988     LocationInfo info = null;
    989
    990     if ((eventFileName != null) || (className != null) || (methodName != null)
    991         || (lineNumber != null)) {
    992       info = new LocationInfo(eventFileName, className, methodName, lineNumber);
    993     } else {
    994       info = LocationInfo.NA_LOCATION_INFO;
    995     }
    996
    997     LoggingEvent event = new LoggingEvent(null,
    998             logger, timeStamp, levelImpl, message,
    999             threadName,
    1000             new ThrowableInformation(exception),
    1001             ndc,
    1002             info,
    1003             properties);
    1004
    1005     return event;
    1006   }
    1007
    1008 //  public static void main(String[] args) {
    1009 //    org.apache.log4j.Logger rootLogger = org.apache.log4j.Logger.getRootLogger();
    1010 //    org.apache.log4j.ConsoleAppender appender = new org.apache.log4j.ConsoleAppender(new org.apache.log4j.SimpleLayout());
    1011 //    appender.setName("console");
    1012 //    rootLogger.addAppender(appender);
    1013 //    LogFilePatternReceiver test = new LogFilePatternReceiver();
    1014 //    org.apache.log4j.spi.LoggerRepository repo = new org.apache.log4j.LoggerRepositoryExImpl(org.apache.log4j.LogManager.getLoggerRepository());
    1015 //    test.setLoggerRepository(repo);
    1016 //    test.setLogFormat("PROP(RELATIVETIME) [THREAD] LEVEL LOGGER * - MESSAGE");
    1017 //    test.setTailing(false);
    1018 //    test.setAppendNonMatches(true);
    1019 //    test.setTimestampFormat("yyyy-MM-d HH:mm:ss,SSS");
    1020 //    test.setFileURL("file:///C:/log/test.log");
    1021 //    test.activateOptions();
    1022 //  }
    1023
    1024     /**
    1025    * Close the reader.
    1026    */
    1027   public void shutdown() {
    1028     getLogger().info(getPath() + " shutdown");
    1029     active = false;
    1030     try {
    1031       if (reader != null) {
    1032         reader.close();
    1033         reader = null;
    1034       }
    1035     } catch (IOException ioe) {
    1036       ioe.printStackTrace();
    1037     }
    1038   }
    1039
    1040     /**
    1041    * Read and process the log file.
    1042    */
    1043   public void activateOptions() {
    1044     getLogger().info("activateOptions");
    1045     active = true;
    1046 	Runnable runnable = new Runnable() {
    1047 	  public void run() {
    1048         initialize();
    1049             while (reader == null) {
    1050                 getLogger().info("attempting to load file: " + getFileURL());
    1051                 try {
    1052                     reader = new InputStreamReader(new URL(getFileURL()).openStream(), "UTF-8");
    1053                 } catch (FileNotFoundException fnfe) {
    1054                     getLogger().info("file not available - will try again");
    1055                     synchronized (this) {
    1056                         try {
    1057                             wait(MISSING_FILE_RETRY_MILLIS);
    1058                         } catch (InterruptedException ie) {}
    1059                     }
    1060                 } catch (IOException ioe) {
    1061                     getLogger().warn("unable to load file", ioe);
    1062                     return;
    1063                 }
    1064             }
    1065             try {
    1066                 BufferedReader bufferedReader = new BufferedReader(reader);
    1067                 createPattern();
    1068                 do {
    1069                     process(bufferedReader);
    1070                     try {
    1071                         synchronized (this) {
    1072                             wait(waitMillis);
    1073                         }
    1074                     } catch (InterruptedException ie) {}
    1075                     if (tailing) {
    1076                       getLogger().debug("tailing file");
    1077                     }
    1078                 } while (tailing);
    1079
    1080             } catch (IOException ioe) {
    1081                 //io exception - probably shut down
    1082                 getLogger().info("stream closed");
    1083             }
    1084             getLogger().debug("processing " + path + " complete");
    1085             shutdown();
    1086           }
    1087         };
    1088         if(useCurrentThread) {
    1089             runnable.run();
    1090         }else {
    1091             new Thread(runnable, "LogFilePatternReceiver-"+getName()).start();
    1092         }
    1093     }
    1094 }