Index: ivy.xml =================================================================== --- ivy.xml (revision 20622) +++ ivy.xml (working copy) @@ -32,5 +32,7 @@ + + Index: main/plugin-site/plugin.xml =================================================================== --- main/plugin-site/plugin.xml (revision 20622) +++ main/plugin-site/plugin.xml (working copy) @@ -188,4 +188,108 @@ + + + + + + + + CONFIG_CACHE_MONITORING_DATASOURCE_JDBC_DRIVER_DESCRIPTION + + + + com.mysql.jdbc.Driver + CONFIG_CACHE_MONITORING_CATEGORY + CONFIG_CACHE_MONITORING_DATASOURCE_JDBC_GROUP + 10 + + + + CONFIG_CACHE_MONITORING_DATASOURCE_JDBC_URL_DESCRIPTION + + + + jdbc:mysql://servername/basename + CONFIG_CACHE_MONITORING_CATEGORY + CONFIG_CACHE_MONITORING_DATASOURCE_JDBC_GROUP + 20 + + + + CONFIG_CACHE_MONITORING_DATASOURCE_JDBC_USER_DESCRIPTION + username + CONFIG_CACHE_MONITORING_CATEGORY + CONFIG_CACHE_MONITORING_DATASOURCE_JDBC_GROUP + 30 + + + + CONFIG_CACHE_MONITORING_DATASOURCE_JDBC_PASSWORD_DESCRIPTION + password + CONFIG_CACHE_MONITORING_CATEGORY + CONFIG_CACHE_MONITORING_DATASOURCE_JDBC_GROUP + 40 + + + + + + + + + + + + + + + + + + CONFIG_CACHE_MONITORING_HANDLING_ENABLE_DESCRIPTION + + + + false + CONFIG_CACHE_MONITORING_CATEGORY + CONFIG_CACHE_MONITORING_HANDLING_GROUP + + + + + + + + + + + + + + + CONFIG_CACHE_MONITORING_APACHE_LOG_PATHS_DESCRIPTION + + ^([^,]*[^\s,][^,]*,)*[^,]*[^\s,][^,]*$ + CONFIG_CACHE_MONITORING_APACHE_LOG_PATHS_INVALID + + CONFIG_CACHE_MONITORING_CATEGORY + CONFIG_CACHE_MONITORING_APACHE_GROUP + + + + + + + Index: main/plugin-site/i18n/messages_fr.xml =================================================================== --- main/plugin-site/i18n/messages_fr.xml (revision 20622) +++ main/plugin-site/i18n/messages_fr.xml (working copy) @@ -26,4 +26,26 @@ Indiquez ici le nombre maximum de caractères du nom des fichiers dans le cache.<br/><b>Attention !</b> Ne modifiez cette valeur que si vous êtes certain d'en comprendre les implications. Une mauvaise valeur peut empêcher les pages au nom trop long d'être affichées, ou réduire les performances du cache. Cache Répertoires + + + Monitoring Cache + + + Connexion à la base de données de monitoring + Pilote + Nom complet de la classe driver JDBC à charger + Url + Url JDBC de connexion au serveur de base de données + Utilisateur + Nom d'utilisateur à utiliser lors de la connexion au serveur de base de données + Mot de passe + Mot de passe associé au nom d'utilisateur à utiliser lors de la connexion au serveur de base de données + + + Fichiers de log Apache + Chemins vers les fichiers + Saisir la liste des fichiers de journalisation des accès d'Apache à monitorer. Les différents chemins doivent être séparés par des virgules. Ces fichiers serviront à produire des statistiques significatives sur l'utilisation des caches. + Entrée invalide. La liste des chemins doit être séparé par des virgules. Index: main/plugin-site/i18n/messages_en.xml =================================================================== --- main/plugin-site/i18n/messages_en.xml (revision 20622) +++ main/plugin-site/i18n/messages_en.xml (working copy) @@ -26,4 +26,26 @@ Specify a maximum length for the cache file names. <br/><b>Be careful!</b> Don't modify this value unless you're sure you understand what it implies. An ill-adapted value can prevent pages with long names for being displayed, or decrease cache performances. Cache Directories + + + Cache Monitoring + + + Monitoring database connection + Driver + JDBC Driver fully qualified class name + Url + JDBC URL for connecting to the database + User + User name for connecting to the database + Password + Password associated with the user name for connecting to the database + + + Apache access logs + Log file paths + Apache to the apache access logs to be monitored in order the produce meaningful statistics + Invalid value. It should be a comma-separated list of paths Index: main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/access/ResourceAccess.java =================================================================== --- main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/access/ResourceAccess.java (revision 0) +++ main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/access/ResourceAccess.java (revision 0) @@ -0,0 +1,44 @@ +/* + * Copyright 2013 Anyware Services + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.ametys.plugins.site.cache.monitoring.process.access; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; + +/** + * Monitored resources. Each access to theses resources is monitored in order to + * be able to analyze cache efficiency and to calculate meaningful statistics + */ +public interface ResourceAccess +{ + /** + * Creates the {@link PreparedStatement} used when doing the SQL insert + * query. + * @param connection The running connection + * @return The prepares statement ready to configure + * @throws SQLException + */ + public PreparedStatement getInsertStatement(Connection connection) throws SQLException; + + /** + * Configures the insert sql statement + * @param stmt The insert sql statement + * @throws SQLException + */ + public void configureInsertStatement(PreparedStatement stmt) throws SQLException; +} Index: main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/access/ResourceAccessComponent.java =================================================================== --- main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/access/ResourceAccessComponent.java (revision 0) +++ main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/access/ResourceAccessComponent.java (revision 0) @@ -0,0 +1,175 @@ +/* + * Copyright 2013 Anyware Services + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.ametys.plugins.site.cache.monitoring.process.access; + +import java.sql.BatchUpdateException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.util.List; + +import org.apache.avalon.framework.activity.Initializable; +import org.apache.avalon.framework.component.Component; +import org.apache.avalon.framework.logger.AbstractLogEnabled; +import org.joda.time.Duration; +import org.joda.time.format.PeriodFormat; + +import org.ametys.plugins.site.cache.monitoring.Constants; +import org.ametys.runtime.config.Config; +import org.ametys.runtime.datasource.ConnectionHelper; + +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.ListMultimap; + +/** + * The RessourceAccessMonitor collects the resources that have been requested, + * and export them into a database. + */ +public class ResourceAccessComponent extends AbstractLogEnabled implements Initializable, Component +{ + /** Avalon ROLE. */ + public static final String ROLE = ResourceAccessComponent.class.getName(); + + /** List of pending {@link ResourceAccess} waiting to be exported to the database. */ + protected ListMultimap _pendingRecords = LinkedListMultimap.create(); + + private boolean _enabled; + + @Override + public void initialize() throws Exception + { + _enabled = Config.getInstance().getValueAsBoolean("front.cache.monitoring.schedulers.enable"); + } + + /** + * Add a new {@link ResourceAccess} to the monitored resources. + * @param ra The resource access object. + */ + public void addAccessRecord(ResourceAccess ra) + { + if (!_enabled) + { + return; + } + + synchronized (this) + { + _pendingRecords.put(ra.getClass().getName(), ra); + } + } + + /** + * Call this method to transfer pendings resource access from memory to database + * This is normally called by a scheduler + */ + public void exportPendings() + { + if (!_enabled) + { + return; + } + + long start = 0; + int successCount = 0; + ListMultimap resourcesAccesstoExport; + + if (getLogger().isDebugEnabled()) + { + getLogger().debug("Start to insert pending records."); + start = System.currentTimeMillis(); + } + + // Copy the current pending records by switching references + synchronized (this) + { + resourcesAccesstoExport = _pendingRecords; + _pendingRecords = LinkedListMultimap.create(); + } + + // Export + if (!resourcesAccesstoExport.isEmpty()) + { + Connection connection = null; + try + { + connection = ConnectionHelper.getConnection(Constants.MONITORING_DATASOURCE_POOLNAME); + successCount = _fillDatabase(connection, resourcesAccesstoExport); + } + catch (Exception e) + { + throw new RuntimeException(e); + } + finally + { + ConnectionHelper.cleanup(connection); + } + } + + if (getLogger().isDebugEnabled()) + { + String durationStr = PeriodFormat.getDefault().print(new Duration(System.currentTimeMillis() - start).toPeriod()); + getLogger().debug(String.format("%s/%s pending records exported into database in %s", successCount, resourcesAccesstoExport.size(), durationStr)); + } + } + + private int _fillDatabase(Connection connection, ListMultimap resourcesAccess) + { + int success = 0; + for (String resourceAccessType : resourcesAccess.keySet()) + { + success += _fillDatabase(connection, resourcesAccess.get(resourceAccessType)); + } + return success; + } + + private int _fillDatabase(Connection connection, List resourcesAccess) + { + PreparedStatement stmt = null; + try + { + if (resourcesAccess.size() > 0) + { + stmt = resourcesAccess.get(0).getInsertStatement(connection); + + for (ResourceAccess ra : resourcesAccess) + { + ra.configureInsertStatement(stmt); + stmt.addBatch(); + } + + return stmt.executeBatch().length; + } + + return 0; + } + catch (BatchUpdateException e) + { + getLogger().error("Batch exception while inserting new records to the database", e); + // more detailed treatment is possible through e.getUpdateCounts() + return 0; + } + catch (SQLException e) + { + getLogger().error("SQLException exception while inserting new records to the database", e); + return 0; + } + finally + { + ConnectionHelper.cleanup(stmt); + } + } +} Index: main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/access/HTTPServerAccessLogImporter.java =================================================================== --- main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/access/HTTPServerAccessLogImporter.java (revision 0) +++ main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/access/HTTPServerAccessLogImporter.java (revision 0) @@ -0,0 +1,281 @@ +package org.ametys.plugins.site.cache.monitoring.process.access; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.List; +import java.util.Locale; + +import org.apache.avalon.framework.activity.Disposable; +import org.apache.avalon.framework.component.Component; +import org.apache.avalon.framework.configuration.Configurable; +import org.apache.avalon.framework.configuration.Configuration; +import org.apache.avalon.framework.configuration.ConfigurationException; +import org.apache.avalon.framework.logger.LogEnabled; +import org.apache.avalon.framework.logger.Logger; +import org.apache.avalon.framework.service.ServiceException; +import org.apache.avalon.framework.service.ServiceManager; +import org.apache.avalon.framework.service.Serviceable; +import org.apache.commons.io.FilenameUtils; +import org.joda.time.Duration; +import org.joda.time.format.PeriodFormat; + +import org.ametys.plugins.site.cache.monitoring.process.access.impl.HTTPServerResourceAccess; +import org.ametys.runtime.config.Config; +import org.ametys.runtime.util.StringUtils; + + +/** + * Import HTTP server access log and pass them to the resource access monitor + */ +public class HTTPServerAccessLogImporter implements Component, Configurable, Serviceable, Disposable, LogEnabled +{ + /** Avalon ROLE. */ + public static final String ROLE = HTTPServerAccessLogImporter.class.getName(); + + /** Logger */ + protected Logger _logger; + + /** The resource access monitoring component */ + protected ResourceAccessComponent _resourceAccessComponent; + + /** + * Date of the initialization of the component, to ensure that only newer + * log entries are importer + */ + protected Date _initializationDate; + + private List _logFileImporters; + + private boolean _enabled; + + @Override + public void service(ServiceManager manager) throws ServiceException + { + _resourceAccessComponent = (ResourceAccessComponent) manager.lookup(ResourceAccessComponent.ROLE); + } + + @Override + public void enableLogging(Logger logger) + { + _logger = logger; + } + + @Override + public void configure(Configuration configuration) throws ConfigurationException + { + _enabled = Config.getInstance().getValueAsBoolean("front.cache.monitoring.schedulers.enable"); + if (!_enabled) + { + return; + } + + _initializationDate = new Date(); + _logFileImporters = new ArrayList(); + + String frontConfig = Config.getInstance().getValueAsString("front.cache.monitoring.apache.log.paths"); + Collection logFilePaths = StringUtils.stringToCollection(frontConfig); + + configureLogFiles(logFilePaths); + + initializeLogFileImporters(); + } + + @Override + public void dispose() + { + _initializationDate = null; + _logFileImporters = null; + } + + /** + * Scan the log files for each site importer. + */ + public synchronized void scanLogFiles() + { + if (!_enabled) + { + return; + } + + for (LogFileImporter importer : _logFileImporters) + { + importer.importEntries(); + } + } + + private void configureLogFiles(Collection paths) + { + for (String path : paths) + { + String dirLog = FilenameUtils.getFullPathNoEndSeparator(path); + String logFilename = FilenameUtils.getName(path); + File logFile = new File(dirLog, logFilename); + + _logFileImporters.add(new LogFileImporter(logFile)); + } + } + + /** + * Initialize the log file importers. This method must not be called during + * the initialization of the component. Instead the call should be delayed, + * and made during the first import attempt. + */ + private void initializeLogFileImporters() + { + for (LogFileImporter importer : _logFileImporters) + { + importer.initialize(); + } + } + + /** + * A log file importer. + * This class is able to import logs from an access log file. + */ + protected final class LogFileImporter + { + private final File _file; + // cached date format per importer, avoid synchronization issues while staying efficient + private final DateFormat _df = new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss Z", Locale.US); + + private BufferedReader _br; + private boolean _initialized; + + /** + * Ctor + * @param logFile + */ + protected LogFileImporter(File logFile) + { + _file = logFile; + } + + /** + * Initialize the log file importer. + */ + protected synchronized void initialize() + { + if (_initialized) + { + return; + } + + try + { + _br = new BufferedReader(new FileReader(_file)); + skipEntriesUntilEOS(); + + _initialized = true; + + if (_logger.isInfoEnabled()) + { + String msg = String.format("The log file importer for the file '%s' is now initialized", _file); + _logger.info(msg); + } + } + catch (IOException e) + { + _logger.error("Exception when initializing the LogFileImporter for the file : '" + _file + "'", e); + } + } + + /** + * Import new entries from the apache log file of this importer + */ + protected synchronized void importEntries() + { + if (!_initialized) + { + _logger.error("LogFileImporter not initialized. Unable to import the new entries for the file : '" + _file + "'"); + return; + } + + try + { + scanLogEntries(); + } + catch (IOException e) + { + _logger.error("IOException while importing apache access log.", e); + } + catch (Exception e) + { + _logger.error("Unexpected exception while importing apache access log.", e); + } + } + + /** + * Skip to the end of the buffered stream + * @throws IOException + */ + private void skipEntriesUntilEOS() throws IOException + { + long start = System.currentTimeMillis(); + + while (_br.readLine() != null) + { + // do nothing, just consume the buffered reader. + } + + if (_logger.isDebugEnabled()) + { + String duration = PeriodFormat.getDefault().print(new Duration(System.currentTimeMillis() - start).toPeriod()); + String msg = String.format("It tooks %s to consume the whole log file '%s'", duration, _file); + _logger.debug(msg); + } + } + + private void scanLogEntries() throws IOException + { + int scanned = 0; + long start = System.currentTimeMillis(); + + // Scan the new lines of the log file line by line. + // For each line, create a new HTTPServerResourceAccess, and pass it to + // the resourceAccessMonitor component if this entry must be + // persisted. + boolean eosReached = false; + while (_br.ready() && !eosReached) + { + String entry = _br.readLine(); + if (entry != null) + { + HTTPServerResourceAccess r = HTTPServerResourceAccess.createRecord(entry, _df); + if (r != null) + { + if (r.isOfInterest(_initializationDate)) + { + _resourceAccessComponent.addAccessRecord(r); + scanned++; + } + else + { + if (_logger.isDebugEnabled()) + { + _logger.debug(String.format("This apache access log entry has been filtered out : %s", r)); + } + } + } + } + else + { + // End of stream has been reached + eosReached = true; + } + } + + if (_logger.isDebugEnabled()) + { + String durationStr = PeriodFormat.getDefault().print(new Duration(System.currentTimeMillis() - start).toPeriod()); + _logger.debug(String.format("%s log entry(ies) scanned in %s", scanned, durationStr)); + } + } + } +} Index: main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/access/impl/HTTPServerResourceAccess.java =================================================================== --- main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/access/impl/HTTPServerResourceAccess.java (revision 0) +++ main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/access/impl/HTTPServerResourceAccess.java (revision 0) @@ -0,0 +1,194 @@ +package org.ametys.plugins.site.cache.monitoring.process.access.impl; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.text.DateFormat; +import java.text.ParseException; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.avalon.framework.logger.Logger; +import org.apache.commons.lang.BooleanUtils; +import org.apache.commons.lang.StringUtils; + +import org.ametys.plugins.site.cache.monitoring.Constants; +import org.ametys.plugins.site.cache.monitoring.process.access.ResourceAccess; +import org.ametys.runtime.util.LoggerFactory; + +/** + * Apache resource access. Represent an access to a resource from Apache. + * These objects are created will parsing the Apache access logs. + */ +public class HTTPServerResourceAccess implements ResourceAccess +{ + /** logger */ + protected static final Logger _LOGGER = LoggerFactory.getLoggerFor(HTTPServerResourceAccess.class); + + private static final Pattern __PATTERN; + static + { + final String sPattern = "^([A-Za-z0-9@-]+) (\\S+) (\\S+) \\S+ .* \\[([^\\]]+)\\] \"([^\"]+)\" (\\d{3})/(\\d{3}) [\\d-]\\d* (-|1) \"([^\"]+)\" \"([^\"]+)\"$"; + __PATTERN = Pattern.compile(sPattern); + } + + private enum Field + { + UNIQUE_ID, + SITE, + REMOTE_HOST_NAME, + DATE, + HTTP_METHOD, + HTTP_PATH, + HTTP_QUERY_STRING, + HTTP_PROTOCOL, + ORI_STATUS_CODE, + RET_STATUS_CODE, + CACHE_HIT, + REFERER, + USER_AGENT + } + + private final String _uniqueID; + private final String _site; + private final String _remoteHostname; + private final Date _date; + private final String _httpMethod; + private final String _httpPath; + private final String _httpQueryString; + private final String _httpProtocol; + private final String _originalStatusCode; + private final String _returnedStatusCode; + private final boolean _cacheHit; + private final String _referer; + private final String _userAgent; + + /** + * Ctor + * @param params + */ + protected HTTPServerResourceAccess(Map params) + { + _uniqueID = (String) params.get(Field.UNIQUE_ID); + _site = (String) params.get(Field.SITE); + _remoteHostname = (String) params.get(Field.REMOTE_HOST_NAME); + _date = (Date) params.get(Field.DATE); + _httpMethod = (String) params.get(Field.HTTP_METHOD); + _httpPath = (String) params.get(Field.HTTP_PATH); + String qs = (String) params.get(Field.HTTP_QUERY_STRING); + _httpQueryString = StringUtils.defaultIfEmpty(qs, "-"); + _httpProtocol = (String) params.get(Field.HTTP_PROTOCOL); + _originalStatusCode = (String) params.get(Field.ORI_STATUS_CODE); + _returnedStatusCode = (String) params.get(Field.RET_STATUS_CODE); + _cacheHit = (Boolean) params.get(Field.CACHE_HIT); + _referer = (String) params.get(Field.REFERER); + _userAgent = (String) params.get(Field.USER_AGENT); + } + + /** + * Create a new record instance + * @param entry the server access log entry + * @param df + * @return the created record + */ + public static HTTPServerResourceAccess createRecord(String entry, DateFormat df) + { + Matcher m = __PATTERN.matcher(entry); + + if (m.matches()) + { + boolean success = true; + + Map params = new HashMap(); + params.put(Field.UNIQUE_ID, m.group(1)); + params.put(Field.SITE, m.group(2)); + params.put(Field.REMOTE_HOST_NAME, m.group(3)); + + try + { + params.put(Field.DATE, df.parse(m.group(4))); + } + catch (NumberFormatException e) + { + success = false; + + // Could happen if there is a synchronization issue with the + // DateFormat instance used to parse the date. + String msg = "NumberFormatException when trying to parse the resource from the apache access logs.\nInput catched string to be parsed '%s'"; + _LOGGER.error(String.format(msg, m.group(4))); + } + catch (ParseException e) + { + _LOGGER.error("Error while parsing the a date from the apache access logs"); + success = false; + } + + String[] httpReq = StringUtils.split(m.group(5), ' '); + params.put(Field.HTTP_METHOD, httpReq[0]); + params.put(Field.HTTP_PATH, StringUtils.substringBefore(httpReq[1], "?")); + params.put(Field.HTTP_QUERY_STRING, StringUtils.substringAfter(httpReq[1], "?")); + params.put(Field.HTTP_PROTOCOL, httpReq[2]); + + params.put(Field.ORI_STATUS_CODE, m.group(6)); + params.put(Field.RET_STATUS_CODE, m.group(7)); + params.put(Field.CACHE_HIT, !"1".equals(m.group(8))); + params.put(Field.REFERER, m.group(9)); + params.put(Field.USER_AGENT, m.group(10)); + + if (success) + { + return new HTTPServerResourceAccess(params); + } + } + + _LOGGER.error("Access log entry does not match the pattern."); + return null; + } + + @Override + public PreparedStatement getInsertStatement(Connection connection) throws SQLException + { + return connection.prepareStatement("INSERT INTO " + Constants.__SQL_TABLE_NAME_HTTPSERVER_ACCESS + " (Unique_Id, Site, Request_Date, Method, Path, Query_String, Ori_Status_Code, Ret_Status_Code, Cache_Hit, Created_At) values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?)"); + } + + @Override + public void configureInsertStatement(PreparedStatement stmt) throws SQLException + { + stmt.setString(1, _uniqueID); + stmt.setString(2, _site); + stmt.setTimestamp(3, new Timestamp(_date.getTime())); + stmt.setString(4, _httpMethod); + stmt.setString(5, _httpPath); + stmt.setString(6, _httpQueryString); + stmt.setString(7, _originalStatusCode); + stmt.setString(8, _returnedStatusCode); + stmt.setInt(9, BooleanUtils.toInteger(_cacheHit)); + stmt.setTimestamp(10, new Timestamp(System.currentTimeMillis())); + } + + /** + * Indicates if this record should be persisted in the database. + * If it returns false, it means that this record must be filtered out and must not be inserted into the database. + * @param date The date at which the apache log importer has been started. + * @return True is this record is newer than the date passed as an argument. + */ + public boolean isOfInterest(Date date) + { + return date.before(_date); + } + + @Override + public String toString() + { + final StringBuilder sb = new StringBuilder(); + sb.append('[').append(_uniqueID).append("] ").append(_site).append(' ').append(_date); + sb.append(' ').append(_httpMethod).append(' ').append(_httpPath).append(StringUtils.isNotEmpty(_httpQueryString) ? '?' + _httpQueryString : "").append(' '); + sb.append(_originalStatusCode).append('/').append(_returnedStatusCode); + sb.append(' ').append("cache-hit:").append(_cacheHit); + return sb.toString(); + } +} Index: main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/access/impl/FrontResourceAccess.java =================================================================== --- main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/access/impl/FrontResourceAccess.java (revision 0) +++ main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/access/impl/FrontResourceAccess.java (revision 0) @@ -0,0 +1,89 @@ +package org.ametys.plugins.site.cache.monitoring.process.access.impl; + +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; + +import org.apache.commons.lang.BooleanUtils; +import org.apache.commons.lang.StringUtils; + +import org.ametys.plugins.site.cache.monitoring.Constants; +import org.ametys.plugins.site.cache.monitoring.process.access.ResourceAccess; + +/** + * Front resource access. Represent an access to a resource from the Front-office. + */ +public class FrontResourceAccess implements ResourceAccess +{ + private final String _uniqueID; + private final String _internalUuid; + private final String _site; + private final String _path; + private boolean _cacheable; + private boolean _cacheHit1; + private boolean _cacheHit2; + + /** + * Ctor + * @param uniqueID + * @param internalUuid + * @param site + * @param path + */ + public FrontResourceAccess(String uniqueID, String internalUuid, String site, String path) + { + _uniqueID = StringUtils.defaultIfEmpty(uniqueID, "-"); + _internalUuid = StringUtils.defaultIfEmpty(internalUuid, "-"); + _site = StringUtils.defaultIfEmpty(site, "-"); + _path = StringUtils.substringBefore(path, "?"); + } + + @Override + public PreparedStatement getInsertStatement(Connection connection) throws SQLException + { + return connection.prepareStatement("INSERT INTO " + Constants.__SQL_TABLE_NAME_FRONT_ACCESS + " (Unique_Id, Internal_Uuid, Site, Ametys_Path, Cacheable, Cache_Hit_1, Cache_Hit_2, Created_At) values (?, ?, ?, ?, ?, ?, ?, ?)"); + } + + @Override + public void configureInsertStatement(PreparedStatement stmt) throws SQLException + { + stmt.setString(1, _uniqueID); + stmt.setString(2, _internalUuid); + stmt.setString(3, _site); + stmt.setString(4, _path); + stmt.setInt(5, BooleanUtils.toInteger(_cacheable)); + stmt.setInt(6, BooleanUtils.toInteger(_cacheHit1)); + stmt.setInt(7, BooleanUtils.toInteger(_cacheHit2)); + stmt.setTimestamp(8, new Timestamp(System.currentTimeMillis())); + } + + /** + * Set the resource as cacheable or not. + * @param cacheable + */ + public void setCacheable(boolean cacheable) + { + _cacheable = cacheable; + } + + /** + * Set the first cache hit to true/false + * @param hit + */ + public void setCacheHit1(boolean hit) + { + _cacheHit1 = hit; + } + + /** + * Set the second cache hit to true/false + * @param hit + */ + public void setCacheHit2(boolean hit) + { + _cacheHit2 = hit; + } + + +} Index: main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/FrontCacheMonitoringScheduler.java =================================================================== --- main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/FrontCacheMonitoringScheduler.java (revision 0) +++ main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/process/FrontCacheMonitoringScheduler.java (revision 0) @@ -0,0 +1,144 @@ +/* + * Copyright 2013 Anyware Services + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.ametys.plugins.site.cache.monitoring.process; + +import java.util.Calendar; +import java.util.GregorianCalendar; +import java.util.Timer; +import java.util.TimerTask; + +import org.apache.avalon.framework.activity.Disposable; +import org.apache.avalon.framework.activity.Initializable; +import org.apache.avalon.framework.logger.LogEnabled; +import org.apache.avalon.framework.logger.Logger; +import org.apache.avalon.framework.service.ServiceException; +import org.apache.avalon.framework.service.ServiceManager; +import org.apache.avalon.framework.service.Serviceable; +import org.joda.time.Duration; +import org.joda.time.format.PeriodFormat; + +import org.ametys.plugins.site.cache.monitoring.process.access.HTTPServerAccessLogImporter; +import org.ametys.plugins.site.cache.monitoring.process.access.ResourceAccessComponent; +import org.ametys.runtime.config.Config; + +/** + * Component responsible of the import of Apache access log file and the + * monitoring of the access of the resources. + */ +public class FrontCacheMonitoringScheduler extends TimerTask implements Initializable, Serviceable, Disposable, LogEnabled +{ + /** Logger */ + protected Logger _logger; + + /** Timer. */ + protected Timer _timer; + + /** Resource Access Monitor */ + protected ResourceAccessComponent _resourceAccessComponent; + + /** Apache access logs importer */ + protected HTTPServerAccessLogImporter _apacheLogImporter; + + @Override + public void service(ServiceManager manager) throws ServiceException + { + _apacheLogImporter = (HTTPServerAccessLogImporter) manager.lookup(HTTPServerAccessLogImporter.ROLE); + _resourceAccessComponent = (ResourceAccessComponent) manager.lookup(ResourceAccessComponent.ROLE); + } + + @Override + public void enableLogging(Logger logger) + { + _logger = logger; + } + + @Override + public void initialize() throws Exception + { + boolean enabled = Config.getInstance().getValueAsBoolean("front.cache.monitoring.schedulers.enable"); + if (!enabled) + { + return; + } + + // Daemon thread + _timer = new Timer("FrontCacheMonitoringScheduler", true); + + // The task is executed each hour, starting at the next hour if not too late. + GregorianCalendar calendar = new GregorianCalendar(); + int minute = calendar.get(Calendar.MINUTE); + + // If 3h55, will starts at 4h. + // If 3h56, will starts at 5h + if (minute <= 55) + { + calendar.add(Calendar.HOUR_OF_DAY, 1); + } + else + { + calendar.add(Calendar.HOUR_OF_DAY, 2); + } + + calendar.set(Calendar.MINUTE, 0); + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MILLISECOND, 0); + + // TODO uncomment after testing +// _timer.scheduleAtFixedRate(this, calendar.getTime(), 60 * 60 * 1000); + _timer.scheduleAtFixedRate(this, 15 * 1000, 60 * 1000); + + if (_logger.isInfoEnabled()) + { + _logger.info("Front cache monitoring scheduler : The process will run each hour, starting " + calendar.getTime()); + } + } + + @Override + public void dispose() + { + cancel(); + _timer.cancel(); + _logger = null; + } + + @Override + public void run() + { + if (_logger.isInfoEnabled()) + { + _logger.info("Starting the front cache monitoring process."); + } + + // Import apache logs. + long start = System.currentTimeMillis(); + _apacheLogImporter.scanLogFiles(); + long endImport = System.currentTimeMillis(); + + // Then export pending records. + _resourceAccessComponent.exportPendings(); + long end = System.currentTimeMillis(); + + if (_logger.isInfoEnabled()) + { + String totalDuration = PeriodFormat.getDefault().print(new Duration(end - start).toPeriod()); + String importDuration = PeriodFormat.getDefault().print(new Duration(endImport - start).toPeriod()); + String exportDuration = PeriodFormat.getDefault().print(new Duration(end - endImport).toPeriod()); + + _logger.info(String.format("\tThe whole front cache monitoring process took %s [apache log import : %s, export to db: %s]", totalDuration, importDuration, exportDuration)); + } + } +} Index: main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/Constants.java =================================================================== --- main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/Constants.java (revision 0) +++ main/plugin-site/src/org/ametys/plugins/site/cache/monitoring/Constants.java (revision 0) @@ -0,0 +1,31 @@ +/* + * Copyright 2013 Anyware Services + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.ametys.plugins.site.cache.monitoring; + +/** + * Constants for cache monitoring + */ +public interface Constants +{ + /** The name of the SQL pool to access monitoring datasource */ + public static final String MONITORING_DATASOURCE_POOLNAME = "cache.monitoring.datasource"; + + /** The name of the table for front resource access */ + public static final String __SQL_TABLE_NAME_FRONT_ACCESS = "Cache_RA_Front"; + /** The name of the table for httpserver resource access */ + public static final String __SQL_TABLE_NAME_HTTPSERVER_ACCESS = "Cache_RA_HTTPServer"; +} Index: main/workspace-site/src/org/ametys/site/IsPageCacheableAction.java =================================================================== --- main/workspace-site/src/org/ametys/site/IsPageCacheableAction.java (revision 20622) +++ main/workspace-site/src/org/ametys/site/IsPageCacheableAction.java (working copy) @@ -32,6 +32,8 @@ import org.apache.excalibur.source.TraversableSource; import org.apache.excalibur.source.impl.FileSource; +import org.ametys.plugins.site.cache.monitoring.process.access.ResourceAccessComponent; +import org.ametys.plugins.site.cache.monitoring.process.access.impl.FrontResourceAccess; import org.ametys.runtime.util.LoggerFactory; /** @@ -42,16 +44,20 @@ /** The cache access component. */ protected CacheAccessManager _cacheAccess; + /** The resource access monitoring component */ + protected ResourceAccessComponent _resourceAccessComponent; + /** The source resolver. */ protected SourceResolver _resolver; - private Logger _logger = LoggerFactory.getLoggerFor("site.cache.log"); + private final Logger _logger = LoggerFactory.getLoggerFor("site.cache.log"); @Override public void service(ServiceManager serviceManager) throws ServiceException { super.service(serviceManager); _cacheAccess = (CacheAccessManager) serviceManager.lookup(CacheAccessManager.ROLE); + _resourceAccessComponent = (ResourceAccessComponent) serviceManager.lookup(ResourceAccessComponent.ROLE); } @Override @@ -62,35 +68,44 @@ // Block until known if the page is cacheable. boolean cacheable = _cacheAccess.isCacheable(source); - String resourceURI = parameters.getParameter("uri", ""); + Request request = ObjectModelHelper.getRequest(objectModel); if (_logger.isDebugEnabled()) { - // the Apache mod_unique_id set a request header "UNIQUE_ID" - // that we could use to track request in log files - Request request = ObjectModelHelper.getRequest(objectModel); String uniqueId = request.getHeader("UNIQUE_ID"); - if (uniqueId != null) { _logger.debug(uniqueId + " - cacheable : " + cacheable); } } + // Resource monitoring + FrontResourceAccess fra = (FrontResourceAccess) request.getAttribute("PAGE_ACCESS"); + fra.setCacheable(cacheable); + // If the page isn't cacheable, unlock and generate (return null). + String resourceURI = parameters.getParameter("uri", ""); if (!cacheable) { + _resourceAccessComponent.addAccessRecord(fra); + _cacheAccess.unlock(source); return null; } // If cacheable and the resource exists, we don't have to generate it. else if (cacheable && _resourceExists(resolver, resourceURI)) { + fra.setCacheHit2(true); + _resourceAccessComponent.addAccessRecord(fra); + _cacheAccess.unlock(source); return EMPTY_MAP; } // Else, return null: the page will be generated into the cache and unlocked when it's done. + fra.setCacheHit2(false); + _resourceAccessComponent.addAccessRecord(fra); + return null; } catch (Exception e) @@ -111,7 +126,7 @@ boolean exists = false; Source source = null; - try + try { source = resolver.resolveURI(resourceURI); @@ -121,20 +136,20 @@ source = SiteCacheHelper.getHashedFileSource(resolver, (FileSource) source); } - if (source.exists() && (!(source instanceof TraversableSource) || !((TraversableSource) source).isCollection())) + if (source.exists() && (!(source instanceof TraversableSource) || !((TraversableSource) source).isCollection())) { exists = true; } - } - catch (SourceNotFoundException e) + } + catch (SourceNotFoundException e) { // Do not log - } - catch (Exception e) + } + catch (Exception e) { getLogger().warn("Exception resolving resource " + resourceURI, e); } - finally + finally { if (source != null) { Index: main/workspace-site/src/org/ametys/site/BackOfficeRequestHelper.java =================================================================== --- main/workspace-site/src/org/ametys/site/BackOfficeRequestHelper.java (revision 20622) +++ main/workspace-site/src/org/ametys/site/BackOfficeRequestHelper.java (working copy) @@ -99,7 +99,7 @@ } /** - * Build a HttpClient request object that will be sent to the back-office to query the page. + * Build a HttpClient request object that will be sent to the back-office to query the page. * @param objectModel the current object model. * @param page the wanted page path. * @return the HttpClient request, to be sent to the back-office. @@ -193,7 +193,7 @@ return boRequest; } - + /** * Get the site's base server path. * @param url the site url object. @@ -222,7 +222,7 @@ return fullUri.toString(); } - + /** * Get the front-office request's parameters as an HttpClient MultipartEntity, * to be added to a POST back-office request. @@ -312,20 +312,27 @@ boRequest.addHeader("X-Ametys-FO-Login", user); } + // Add apache unique-id + String uuid = (String) request.getAttribute("Monitoring-UUID"); + if (uuid != null) + { + boRequest.addHeader("X-Ametys-FO-UUID", uuid); + } + // Forwarding headers - Enumeration headers = request.getHeaderNames(); + Enumeration headers = request.getHeaderNames(); while (headers.hasMoreElements()) { String headerName = headers.nextElement(); if (__AUTHORIZED_HEADERS.matcher(headerName).matches()) { // forward - Enumeration headerValues = request.getHeaders(headerName); + Enumeration headerValues = request.getHeaders(headerName); while (headerValues.hasMoreElements()) { String headerValue = headerValues.nextElement(); boRequest.addHeader(headerName, headerValue); - } + } } } } @@ -415,7 +422,7 @@ { // the wrapped entity's getContent() decides about repeatability InputStream wrappedin = wrappedEntity.getContent(); - + return new GZIPInputStream(wrappedin); } Index: main/workspace-site/src/org/ametys/site/ResourceExistsAction.java =================================================================== --- main/workspace-site/src/org/ametys/site/ResourceExistsAction.java (revision 20622) +++ main/workspace-site/src/org/ametys/site/ResourceExistsAction.java (working copy) @@ -17,33 +17,67 @@ package org.ametys.site; import java.util.Map; +import java.util.UUID; import org.apache.avalon.framework.logger.Logger; import org.apache.avalon.framework.parameters.Parameters; +import org.apache.avalon.framework.service.ServiceException; +import org.apache.avalon.framework.service.ServiceManager; import org.apache.avalon.framework.thread.ThreadSafe; import org.apache.cocoon.acting.ServiceableAction; +import org.apache.cocoon.environment.ObjectModelHelper; import org.apache.cocoon.environment.Redirector; +import org.apache.cocoon.environment.Request; import org.apache.cocoon.environment.SourceResolver; +import org.apache.commons.lang.StringUtils; import org.apache.excalibur.source.Source; import org.apache.excalibur.source.SourceNotFoundException; import org.apache.excalibur.source.TraversableSource; import org.apache.excalibur.source.impl.FileSource; +import org.ametys.plugins.site.cache.monitoring.process.access.ResourceAccessComponent; +import org.ametys.plugins.site.cache.monitoring.process.access.impl.FrontResourceAccess; +import org.ametys.runtime.config.Config; import org.ametys.runtime.util.LoggerFactory; /** * The cocoon resource exists action but that returns false for folders */ -public class ResourceExistsAction extends ServiceableAction implements ThreadSafe +public class ResourceExistsAction extends ServiceableAction implements ThreadSafe { + /** The resource access monitoring component */ + protected ResourceAccessComponent _resourceAccessComponent; + + // FIXME static final logger? private Logger _logger = LoggerFactory.getLoggerFor("site.cache.log"); @Override + public void service(ServiceManager serviceManager) throws ServiceException + { + super.service(serviceManager); + _resourceAccessComponent = (ResourceAccessComponent) serviceManager.lookup(ResourceAccessComponent.ROLE); + } + + @Override public Map act(Redirector redirector, SourceResolver resolver, Map objectModel, String src, Parameters parameters) throws Exception { String resourceURI = parameters.getParameter("url", src); Source source = null; - try + + // Resource monitoring + // the Apache mod_unique_id set a request header "UNIQUE_ID" + // that we could use to track request in log files + Request request = ObjectModelHelper.getRequest(objectModel); + String uniqueId = request.getHeader("UNIQUE_ID"); + + // Set the random uuid as a request attribute to be added later in the possible requests to the BO. + String uuid = UUID.randomUUID().toString(); + request.setAttribute("Monitoring-UUID", uuid); + + String path = StringUtils.removeStart(resourceURI, Config.getInstance().getValueAsString("org.ametys.site.root")); + FrontResourceAccess fra = new FrontResourceAccess(uniqueId, uuid, parameters.getParameter("site", null), path); + + try { source = resolver.resolveURI(resourceURI); @@ -53,31 +87,40 @@ source = SiteCacheHelper.getHashedFileSource(resolver, (FileSource) source); } - if (source.exists() - && (!(source instanceof TraversableSource) || !((TraversableSource) source).isCollection())) + if (source.exists() + && (!(source instanceof TraversableSource) || !((TraversableSource) source).isCollection())) { if (_logger.isDebugEnabled()) { _logger.debug("Find resource '" + resourceURI + " in cache"); } + + fra.setCacheable(true); + fra.setCacheHit1(true); + _resourceAccessComponent.addAccessRecord(fra); + return EMPTY_MAP; } - } - catch (SourceNotFoundException e) + } + catch (SourceNotFoundException e) { // Do not log } - catch (Exception e) + catch (Exception e) { getLogger().warn("Exception resolving resource " + resourceURI, e); - } - finally + } + finally { - if (source != null) + if (source != null) { resolver.release(source); } } + + fra.setCacheHit1(false); + request.setAttribute("PAGE_ACCESS", fra); + return null; } Index: main/workspace-site/sitemap.xmap =================================================================== --- main/workspace-site/sitemap.xmap (revision 20622) +++ main/workspace-site/sitemap.xmap (working copy) @@ -156,12 +156,14 @@ + +