Index: ivy.xml =================================================================== --- ivy.xml (revision 20303) +++ ivy.xml (working copy) @@ -32,5 +32,7 @@ + + Index: main/plugin-site/plugin.xml =================================================================== --- main/plugin-site/plugin.xml (revision 20303) +++ main/plugin-site/plugin.xml (working copy) @@ -188,4 +188,95 @@ + + + + + + + + CONFIG_CACHE_MONITORING_DATASOURCE_JDBC_DRIVER_DESCRIPTION + + + + com.mysql.jdbc.Driver + CONFIG_CACHE_MONITORING_CATEGORY + CONFIG_CACHE_MONITORING_DATASOURCE_JDBC_GROUP + 10 + + + + CONFIG_CACHE_MONITORING_DATASOURCE_JDBC_URL_DESCRIPTION + + + + jdbc:mysql://servername/basename + CONFIG_CACHE_MONITORING_CATEGORY + CONFIG_CACHE_MONITORING_DATASOURCE_JDBC_GROUP + 20 + + + + CONFIG_CACHE_MONITORING_DATASOURCE_JDBC_USER_DESCRIPTION + username + CONFIG_CACHE_MONITORING_CATEGORY + CONFIG_CACHE_MONITORING_DATASOURCE_JDBC_GROUP + 30 + + + + CONFIG_CACHE_MONITORING_DATASOURCE_JDBC_PASSWORD_DESCRIPTION + password + CONFIG_CACHE_MONITORING_CATEGORY + CONFIG_CACHE_MONITORING_DATASOURCE_JDBC_GROUP + 40 + + + + + + + + + + + + + + + + + + + + + + + + + + CONFIG_CACHE_MONITORING_APACHE_LOG_PATHS_DESCRIPTION + + ^([^,]*[^\s,][^,]*,)*[^,]*[^\s,][^,]*$ + CONFIG_CACHE_MONITORING_APACHE_LOG_PATHS_INVALID + + CONFIG_CACHE_MONITORING_CATEGORY + CONFIG_CACHE_MONITORING_APACHE_GROUP + + + + + + + Index: main/plugin-site/i18n/messages_fr.xml =================================================================== --- main/plugin-site/i18n/messages_fr.xml (revision 20303) +++ main/plugin-site/i18n/messages_fr.xml (working copy) @@ -26,4 +26,26 @@ Indiquez ici le nombre maximum de caractères du nom des fichiers dans le cache.<br/><b>Attention !</b> Ne modifiez cette valeur que si vous êtes certain d'en comprendre les implications. Une mauvaise valeur peut empêcher les pages au nom trop long d'être affichées, ou réduire les performances du cache. Cache Répertoires + + + Monitoring Cache + + + Connexion à la base de données de monitoring + Pilote + Nom complet de la classe driver JDBC à charger + Url + Url JDBC de connexion au serveur de base de données + Utilisateur + Nom d'utilisateur à utiliser lors de la connexion au serveur de base de données + Mot de passe + Mot de passe associé au nom d'utilisateur à utiliser lors de la connexion au serveur de base de données + + + Fichiers de log Apache + Chemins vers les fichiers + Saisir la liste des fichiers de journalisation des accès d'Apache à monitorer. Les différents chemins doivent être séparés par des virgules. Ces fichiers serviront à produire des statistiques significatives sur l'utilisation des caches. + Entrée invalide. La liste des chemins doit être séparé par des virgules. Index: main/plugin-site/i18n/messages_en.xml =================================================================== --- main/plugin-site/i18n/messages_en.xml (revision 20303) +++ main/plugin-site/i18n/messages_en.xml (working copy) @@ -26,4 +26,26 @@ Specify a maximum length for the cache file names. <br/><b>Be careful!</b> Don't modify this value unless you're sure you understand what it implies. An ill-adapted value can prevent pages with long names for being displayed, or decrease cache performances. Cache Directories + + + Cache Monitoring + + + Monitoring database connection + Driver + JDBC Driver fully qualified class name + Url + JDBC URL for connecting to the database + User + User name for connecting to the database + Password + Password associated with the user name for connecting to the database + + + Apache access logs + Log file paths + Apache to the apache access logs to be monitored in order the produce meaningful statistics + Invalid value. It should be a comma-separated list of paths Index: main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/server/ApacheResourceAccess.java =================================================================== --- main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/server/ApacheResourceAccess.java (revision 0) +++ main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/server/ApacheResourceAccess.java (revision 0) @@ -0,0 +1,193 @@ +package org.ametys.plugins.site.cachemonitoring.server; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Timestamp; +import java.text.DateFormat; +import java.text.ParseException; +import java.util.Date; +import java.util.HashMap; +import java.util.Map; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.avalon.framework.logger.Logger; +import org.apache.commons.lang.StringUtils; + +import org.ametys.plugins.site.cachemonitoring.core.ResourceAccess; +import org.ametys.runtime.util.LoggerFactory; + +/** + * Apache resource access. Represent an access to a resource from Apache. + * These objects are created will parsing the Apache access logs. + */ +public class ApacheResourceAccess implements ResourceAccess +{ + /** logger */ + protected static final Logger _LOGGER = LoggerFactory.getLoggerFor(ApacheResourceAccess.class); + + private static final Pattern __PATTERN; + static + { + final String sPattern = "^([A-Za-z0-9@-]+) (\\S+) (\\S+) \\S+ .* \\[([^\\]]+)\\] \"([^\"]+)\" (\\d{3})/(\\d{3}) [\\d-]\\d* (-|1) \"([^\"]+)\" \"([^\"]+)\"$"; + __PATTERN = Pattern.compile(sPattern); + } + + private enum Field + { + UNIQUE_ID, + SITE, + REMOTE_HOST_NAME, + DATE, + HTTP_METHOD, + HTTP_PATH, + HTTP_QUERY_STRING, + HTTP_PROTOCOL, + ORI_STATUS_CODE, + RET_STATUS_CODE, + CACHE_HIT, + REFERER, + USER_AGENT + } + + private final String _uniqueID; + private final String _site; + private final String _remoteHostname; + private final Date _date; + private final String _httpMethod; + private final String _httpPath; + private final String _httpQueryString; + private final String _httpProtocol; + private final String _originalStatusCode; + private final String _returnedStatusCode; + private final boolean _cacheHit; + private final String _referer; + private final String _userAgent; + + /** + * Ctor + * @param params + */ + protected ApacheResourceAccess(Map params) + { + _uniqueID = (String) params.get(Field.UNIQUE_ID); + _site = (String) params.get(Field.SITE); + _remoteHostname = (String) params.get(Field.REMOTE_HOST_NAME); + _date = (Date) params.get(Field.DATE); + _httpMethod = (String) params.get(Field.HTTP_METHOD); + _httpPath = (String) params.get(Field.HTTP_PATH); + _httpQueryString = (String) params.get(Field.HTTP_QUERY_STRING); + _httpProtocol = (String) params.get(Field.HTTP_PROTOCOL); + _originalStatusCode = (String) params.get(Field.ORI_STATUS_CODE); + _returnedStatusCode = (String) params.get(Field.RET_STATUS_CODE); + _cacheHit = (Boolean) params.get(Field.CACHE_HIT); + _referer = (String) params.get(Field.REFERER); + _userAgent = (String) params.get(Field.USER_AGENT); + } + + /** + * Create a new record instance + * @param entry the server access log entry + * @param df + * @return the created record + */ + public static ApacheResourceAccess createRecord(String entry, DateFormat df) + { + Matcher m = __PATTERN.matcher(entry); + + if (m.matches()) + { + boolean success = true; + + Map params = new HashMap(); + params.put(Field.UNIQUE_ID, m.group(1)); + params.put(Field.SITE, m.group(2)); + params.put(Field.REMOTE_HOST_NAME, m.group(3)); + + try + { + params.put(Field.DATE, df.parse(m.group(4))); + } + catch (NumberFormatException e) + { + success = false; + + // Could happen if there is a synchronization issue with the + // DateFormat instance used to parse the date. + String msg = "NumberFormatException when trying to parse the resource from the apache access logs.\nInput catched string to be parsed '%s'"; + _LOGGER.error(String.format(msg, m.group(4))); + } + catch (ParseException e) + { + _LOGGER.error("Error while parsing the a date from the apache access logs"); + success = false; + } + + String[] httpReq = StringUtils.split(m.group(5), ' '); + params.put(Field.HTTP_METHOD, httpReq[0]); + params.put(Field.HTTP_PATH, StringUtils.substringBefore(httpReq[1], "?")); + params.put(Field.HTTP_QUERY_STRING, StringUtils.substringAfter(httpReq[1], "?")); + params.put(Field.HTTP_PROTOCOL, httpReq[2]); + + params.put(Field.ORI_STATUS_CODE, m.group(6)); + params.put(Field.RET_STATUS_CODE, m.group(7)); + params.put(Field.CACHE_HIT, !"1".equals(m.group(8))); + params.put(Field.REFERER, m.group(9)); + params.put(Field.USER_AGENT, m.group(10)); + + if (success) + { + return new ApacheResourceAccess(params); + } + } + + _LOGGER.error("Access log entry does not match the pattern."); + return null; + } + + @Override + public MonitoredResourceType getType() + { + return MonitoredResourceType.APACHE_RESOURCE; + } + + @Override + public void configureSqlInsert(PreparedStatement ps) throws SQLException + { + ps.setString(1, _uniqueID); + ps.setString(2, _site); + ps.setString(3, _remoteHostname); + ps.setTimestamp(4, new Timestamp(_date.getTime())); + ps.setString(5, _httpMethod); + ps.setString(6, _httpPath); + ps.setString(7, _httpQueryString); + ps.setString(8, _httpProtocol); + ps.setString(9, _originalStatusCode); + ps.setString(10, _returnedStatusCode); + ps.setBoolean(11, _cacheHit); + ps.setString(12, _referer); + ps.setString(13, _userAgent); + } + + /** + * Indicates if this record should be persisted in the database. + * If it returns false, it means that this record must be filtered out and must not be inserted into the database. + * @param date The date at which the apache log importer has been started. + * @return True is this record is newer than the date passed as an argument. + */ + public boolean isOfInterest(Date date) + { + return date.before(_date); + } + + @Override + public String toString() + { + final StringBuilder sb = new StringBuilder(); + sb.append('[').append(_uniqueID).append("] ").append(_site).append(' ').append(_date); + sb.append(' ').append(_httpMethod).append(' ').append(_httpPath).append(StringUtils.isNotEmpty(_httpQueryString) ? '?' + _httpQueryString : "").append(' '); + sb.append(_originalStatusCode).append('/').append(_returnedStatusCode); + sb.append(' ').append("cache-hit:").append(_cacheHit); + return sb.toString(); + } +} Index: main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/front/FrontResourceAccess.java =================================================================== --- main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/front/FrontResourceAccess.java (revision 0) +++ main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/front/FrontResourceAccess.java (revision 0) @@ -0,0 +1,84 @@ +package org.ametys.plugins.site.cachemonitoring.front; + +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Types; + +import org.apache.commons.lang.StringUtils; + +import org.ametys.plugins.site.cachemonitoring.core.ResourceAccess; + +/** + * Front resource access. Represent an access to a resource from the Front-office. + */ +public class FrontResourceAccess implements ResourceAccess +{ + private final String _uniqueID; + private final String _internalUuid; + private final String _site; + private final String _path; + private boolean _cacheable; + private Boolean _cacheHit1; + private Boolean _cacheHit2; + + /** + * Ctor + * @param uniqueID + * @param internalUuid + * @param site + * @param path + */ + public FrontResourceAccess(String uniqueID, String internalUuid, String site, String path) + { + _uniqueID = uniqueID; + _internalUuid = internalUuid; + _site = StringUtils.defaultString(site); + _path = StringUtils.substringBefore(path, "?"); + } + + /** + * Set the resource as cacheable or not. + * @param cacheable + */ + public void setCacheable(boolean cacheable) + { + _cacheable = cacheable; + } + + /** + * Set the first cache hit to true/false + * @param hit + */ + public void setCacheHit1(boolean hit) + { + _cacheHit1 = hit; + } + + /** + * Set the second cache hit to true/false + * @param hit + */ + public void setCacheHit2(boolean hit) + { + _cacheHit2 = hit; + } + + @Override + public MonitoredResourceType getType() + { + return MonitoredResourceType.FRONT_RESOURCE; + } + + @Override + public void configureSqlInsert(PreparedStatement ps) throws SQLException + { + ps.setString(1, _uniqueID); + ps.setString(2, _internalUuid); + ps.setString(3, _site); + ps.setString(4, _path); + ps.setBoolean(5, _cacheable); + // Handle null cases, avoid NPE and set NULL into the db in thoses cases. + ps.setObject(6, _cacheHit1, Types.BOOLEAN); + ps.setObject(7, _cacheHit2, Types.BOOLEAN); + } +} Index: main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/CacheMonitoringSqlStatements.java =================================================================== --- main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/CacheMonitoringSqlStatements.java (revision 0) +++ main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/CacheMonitoringSqlStatements.java (revision 0) @@ -0,0 +1,459 @@ +package org.ametys.plugins.site.cachemonitoring.core; + +import java.util.Arrays; + +import org.apache.commons.lang.StringUtils; + +import org.ametys.plugins.site.cachemonitoring.core.ResourceAccess.MonitoredResourceType; +import org.ametys.plugins.site.cachemonitoring.core.ResourceCacheStats.ResourceCacheStatsType; + +/** + * Gather SQL statements related to the cache monitoring process. + */ +public final class CacheMonitoringSqlStatements +{ + /** Datasource id of the cache monitoring datasource */ + public static final String MONITORING_DATASOURCE_ID = "cache.monitoring.datasource"; + + // >--------------------- ACCESS TABLES ------------------------< + + // FIXME Config + // private static final String __APACHE_RESOURCE_ACCESS_TABLE_NAME = Config.SERVER_REQUESTS_TABLE_NAME; + // private static final String __FRONT_RESOURCE_ACCESS_TABLE_NAME = Config.FRONT_REQUESTS_TABLE_NAME; + // private static final String __PAGE_RESOURCE_ACCESS_TABLE_NAME = Config.PAGE_REQUESTS_TABLE_NAME; + // private static final String __PAGE_ELEMENT_RESOURCE_ACCESS_TABLE_NAME = Config.PAGE_ELEMENT_REQUESTS_TABLE_NAME; + + private static final String __APACHE_RESOURCE_ACCESS_TABLE_NAME = "Server_Requests"; + private static final String __FRONT_RESOURCE_ACCESS_TABLE_NAME = "Front_Requests"; + private static final String __PAGE_RESOURCE_ACCESS_TABLE_NAME = "Back_Requests"; + private static final String __PAGE_ELEMENT_RESOURCE_ACCESS_TABLE_NAME = "Back_Page_Element_Requests"; + + + private static final String __INSERT_RESOURCE_ACCESS_SQL = "INSERT IGNORE INTO $tbl_name$ ($insert_cols$) values ($insert_values$)"; + private static final String __TBL_NAME_PATTERN = "$tbl_name$"; + private static final String __INSERT_COLS_CLAUSE_PATTERN = "$insert_cols$"; + private static final String __INSERT_VALUES_CLAUSE_PATTERN = "$insert_values$"; + private static final String __APACHE_INSERT_RESOURCE_COLS_CLAUSE = "Unique_Id, Site, Remote_Host_Name, Request_Date, Method, Path, Query_String, Protocol, Ori_Status_Code, Ret_Status_Code, Cache_Hit, Referer, User_Agent"; + private static final String __FRONT_INSERT_RESOURCE_COLS_CLAUSE = "Unique_Id, Internal_Uuid, Site, Ametys_Path, Cacheable, Cache_Hit_1, Cache_Hit_2"; + private static final String __PAGE_INSERT_RESOURCE_COLS_CLAUSE = "Internal_Uuid, Page_Id, Page_Path, Rendering_Context, Workspace_JCR, Cacheable"; + private static final String __PAGE_ELEMENT_INSERT_RESOURCE_COLS_CLAUSE = "Internal_Uuid, Page_Element_Id, Page_Element_Type, Page_Id, Rendering_Context, Workspace_JCR, Cacheable, Cache_Hit"; + + private static final String __GROUP_FIELDS_PATTERN = "$group_fields$"; + private static final String __COLS_FIELDS_PATTERN = "$cols_fields$"; + private static final String __INTERNAL_GET_RES_PATTERN = "$internal_get_res$"; + private static final String __GET_RESOURCE_TO_PROCESS = "SELECT count(*) AS increment, $cols_fields$ FROM $tbl_name$ T $internal_get_res$ GROUP BY $group_fields$ ORDER BY max(T.Id)"; + + private static final String __APACHE_PROCESS_GROUP_FIELDS = "Site, Path, Cache_Hit"; + private static final String __FRONT_PROCESS_GROUP_FIELDS = "Site, Ametys_Path, Cacheable, Cache_Hit_1, Cache_Hit_2"; + private static final String __FRONT_APACHE_PROCESS_GROUP_FIELDS = "S.Site, S.Path, S.Cache_Hit, T.Site, T.Ametys_Path, T.Cacheable, T.Cache_Hit_1, T.Cache_Hit_2"; + private static final String __PAGE_PROCESS_GROUP_FIELDS = "Page_Id, Page_Path, Rendering_Context, Workspace_JCR, Cacheable"; + private static final String __PAGE_ELEMENT_PROCESS_GROUP_FIELDS = "Page_Element_Id, Page_Id, Rendering_Context, Workspace_JCR, Cacheable, Cache_Hit"; + + private static final String __APACHE_PROCESS_COLS_FIELDS = __APACHE_PROCESS_GROUP_FIELDS; + private static final String __FRONT_PROCESS_COLS_FIELDS = __FRONT_PROCESS_GROUP_FIELDS; + private static final String __FRONT_APACHE_PROCESS_COLS_FIELDS = "S.Site AS S_Site, S.Path AS S_Path, S.Cache_Hit AS S_Cache_Hit, T.Site AS F_Site, T.Ametys_Path AS F_Ametys_Path, T.Cacheable AS F_Cacheable, T.Cache_Hit_1 AS F_Cache_Hit_1, T.Cache_Hit_2 AS F_Cache_Hit_2"; + private static final String __PAGE_PROCESS_COLS_FIELDS = __PAGE_PROCESS_GROUP_FIELDS; + private static final String __PAGE_ELEMENT_PROCESS_COLS_FIELDS = __PAGE_ELEMENT_PROCESS_GROUP_FIELDS; + + private static final String __APACHE_INTERNAL_GET_RES_PATTERN = "WHERE PROCESSED = false AND Cache_Hit = true AND (Ori_Status_Code = 200 OR Ori_Status_Code = 304)"; + private static final String __FRONT_INTERNAL_GET_RES_PATTERN = "WHERE PROCESSED = false AND Unique_Id is NULL"; + private static final String __FRONT_APACHE_INTERNAL_GET_RES_PATTERN = "INNER JOIN " + __APACHE_RESOURCE_ACCESS_TABLE_NAME + " S ON T.Unique_Id = S.Unique_Id AND T.Processed = false AND T.Unique_Id is not NULL AND (S.Ori_Status_Code = 200 OR S.Ori_Status_Code = 304)"; + private static final String __PAGE_INTERNAL_GET_RES_PATTERN = "WHERE PROCESSED = false"; + private static final String __PAGE_ELEMENT_INTERNAL_GET_RES_PATTERN = "WHERE PROCESSED = false"; + + private static final String __GET_MAX_ID_TO_PROCESS = "SELECT max(Id) FROM $tbl_name$ WHERE PROCESSED = false"; + private static final String __GET_MAX_ID_TO_PROCESS_APACHE = "SELECT max(Id) FROM $tbl_name$ WHERE PROCESSED = false AND (Ori_Status_Code = 200 OR Ori_Status_Code = 304)"; + private static final String __GET_MAX_ID_TO_PROCESS_FRONT = "SELECT max(Id) FROM $tbl_name$ WHERE PROCESSED = false AND Unique_Id is NULL"; + private static final String __GET_MAX_ID_TO_PROCESS_FRONT_APACHE = "SELECT max(F.Id) FROM $tbl_name$ AS F INNER JOIN " + __APACHE_RESOURCE_ACCESS_TABLE_NAME + " S ON F.Unique_Id = S.Unique_Id AND F.Processed = false AND F.Unique_Id is not NULL AND (S.Ori_Status_Code = 200 OR S.Ori_Status_Code = 304)"; + + private static final String __MARK_AS_PROCESSED = "UPDATE $tbl_name$ SET PROCESSED = true WHERE Id <= ? AND Processed = false"; + private static final String __MARK_AS_PROCESSED_APACHE = "UPDATE $tbl_name$ AS S LEFT OUTER JOIN " + __FRONT_RESOURCE_ACCESS_TABLE_NAME + " F ON S.Unique_Id = F.Unique_Id SET S.Processed = true WHERE S.Id <= ? AND F.Unique_Id is NULL AND S.Processed = false AND (S.Ori_Status_Code = 200 OR S.Ori_Status_Code = 304)"; + private static final String __MARK_AS_PROCESSED_FRONT = "UPDATE $tbl_name$ SET PROCESSED = true WHERE Id <= ? AND Processed = false AND Unique_Id is NULL"; + private static final String __MARK_AS_PROCESSED_FRONT_APACHE = "UPDATE $tbl_name$ AS F INNER JOIN " + __APACHE_RESOURCE_ACCESS_TABLE_NAME + " S ON F.Unique_Id = S.Unique_Id AND F.Processed = false AND F.Unique_Id is not NULL AND (S.Ori_Status_Code = 200 OR S.Ori_Status_Code = 304) SET F.Processed = true WHERE F.Id <= ?"; + private static final String __MARK_AS_PROCESSED_APACHE_AFTER_FRONT_APACHE = "UPDATE " + __APACHE_RESOURCE_ACCESS_TABLE_NAME + " AS S INNER JOIN " + __FRONT_RESOURCE_ACCESS_TABLE_NAME + " F ON S.Unique_Id = F.Unique_Id AND S.Processed = false AND F.Processed = true SET S.Processed = true"; + + private static final String __PURGE_PROCESSED_SQL = "DELETE FROM $tbl_name$ WHERE PROCESSED = true"; + + // >--------------------- CACHE STATS TABLES ------------------------< + + // FIXME Config + // private static final String __APACHE_CACHE_STATS_TABLE_NAME = Config.SERVER_CACHE_STATS_TABLE_NAME; + // private static final String __FRONT_CACHE_STATS_TABLE_NAME = Config.FRONT_CACHE_STATS_TABLE_NAME; + // private static final String __PAGE_CACHE_STATS_TABLE_NAME = Config.PAGE_CACHE_STATS_TABLE_NAME; + // private static final String __PAGE_ELEMENT_CACHE_STATS_TABLE_NAME = Config.PAGE_ELEMENT_CACHE_STATS_TABLE_NAME; + + private static final String __FRONT_CACHE_STATS_TABLE_NAME = "Cache_Front_Stats"; + private static final String __PAGE_CACHE_STATS_TABLE_NAME = "Cache_Back_Stats"; + private static final String __PAGE_ELEMENT_CACHE_STATS_TABLE_NAME = "Cache_Back_Page_Element_Stats"; + + + private static final String __INSERT_CACHE_STATS = "INSERT INTO $tbl_name$ ($insert_cols$) values ($insert_values$)"; + private static final String __APACHE_INSERT_CACHE_COLS_CLAUSE = "Server_Site, Server_Path, Server_Hits, Server_Cache_Hits"; + private static final String __FRONT_INSERT_CACHE_COLS_CLAUSE = "Front_Site, Front_Path, Front_Cacheable, Front_Hits, Front_Cache_Hits_1, Front_Cache_Hits_2"; + private static final String __FRONT_APACHE_INSERT_CACHE_COLS_CLAUSE = __APACHE_INSERT_CACHE_COLS_CLAUSE + ", " + __FRONT_INSERT_CACHE_COLS_CLAUSE; + private static final String __PAGE_INSERT_CACHE_COLS_CLAUSE = "Page_Id, Page_Path, Rendering_Context, Workspace_JCR, Cacheable, Hits"; + private static final String __PAGE_ELEMENT_INSERT_CACHE_COLS_CLAUSE = "Page_Element_Id, Page_Id, Rendering_Context, Workspace_JCR, Cacheable, Hits, Cache_Hits"; + + private static final String __FIND_CACHE_STATS = "SELECT count(1) FROM $tbl_name$ WHERE $where_clause$"; + private static final String __WHERE_CLAUSE_PATTERN = "$where_clause$"; + private static final String __APACHE_WHERE_CLAUSE = "Server_Site = ? AND Server_Path = ?"; + private static final String __FRONT_WHERE_CLAUSE = "Front_Site = ? AND Front_Path = ? AND Server_Path is NULL"; + private static final String __FRONT_APACHE_WHERE_CLAUSE = __APACHE_WHERE_CLAUSE; + private static final String __PAGE_WHERE_CLAUSE = "Page_Id = ? AND Rendering_Context = ? AND Workspace_JCR = ?"; + private static final String __PAGE_ELEMENT_WHERE_CLAUSE = "Page_Element_Id = ? AND Page_Id = ? AND Rendering_Context = ? AND Workspace_JCR = ?"; + + private static final String __UPDATE_CACHE_STATS = "UPDATE $tbl_name$ SET $set_clause$ WHERE $where_clause$"; + private static final String __SET_CLAUSE_PATTERN = "$set_clause$"; + private static final String __APACHE_SET_CLAUSE = "Server_Hits = Server_Hits + ?, Server_Cache_Hits = Server_Cache_Hits + ?"; + private static final String __FRONT_SET_CLAUSE = "Front_Cacheable = ?, Front_Hits = Front_Hits + ?, Front_Cache_Hits_1 = Front_Cache_Hits_1 + ?, Front_Cache_Hits_2 = Front_Cache_Hits_2 + ?"; + private static final String __FRONT_APACHE_SET_CLAUSE = __APACHE_SET_CLAUSE + ", Front_Site = ?, Front_Path = ?, " + __FRONT_SET_CLAUSE; + private static final String __PAGE_SET_CLAUSE = "Page_Path = ?, Cacheable = ?, Hits = Hits + ?"; + private static final String __PAGE_ELEMENT_SET_CLAUSE = "Cacheable = ?, Hits = Hits + ?, Cache_Hits = Cache_Hits + ?"; + + private CacheMonitoringSqlStatements() + { + // ignore + } + + /** + * Insert resource access + * @param type + * @return The sql query + */ + public static String insertResourceAccessSql(MonitoredResourceType type) + { + String insertCols = null; + switch (type) + { + case APACHE_RESOURCE: + insertCols = __APACHE_INSERT_RESOURCE_COLS_CLAUSE; + break; + case FRONT_RESOURCE: + insertCols = __FRONT_INSERT_RESOURCE_COLS_CLAUSE; + break; + case BACK_PAGE_RESOURCE: + insertCols = __PAGE_INSERT_RESOURCE_COLS_CLAUSE; + break; + case BACK_PAGE_ELEMENT: + insertCols = __PAGE_ELEMENT_INSERT_RESOURCE_COLS_CLAUSE; + break; + default: + throw new IllegalArgumentException("Illegal type. Allowed types are : " + Arrays.asList(MonitoredResourceType.values())); + } + + String[] insertValuesArray = StringUtils.split(insertCols, ", "); + Arrays.fill(insertValuesArray, "?"); + String insertValues = StringUtils.join(insertValuesArray, ", "); + + String sql = StringUtils.replaceOnce(__INSERT_RESOURCE_ACCESS_SQL, __TBL_NAME_PATTERN, getResourceAccessTableName(type)); + sql = StringUtils.replaceOnce(sql, __INSERT_COLS_CLAUSE_PATTERN, insertCols); + sql = StringUtils.replaceOnce(sql, __INSERT_VALUES_CLAUSE_PATTERN, insertValues); + + return sql; + } + + /** + * The purge sql query + * @param type + * @return The sql query + */ + public static String getPurgeSql(MonitoredResourceType type) + { + return StringUtils.replaceOnce(__PURGE_PROCESSED_SQL, __TBL_NAME_PATTERN, getResourceAccessTableName(type)); + } + + private static String getResourceAccessTableName(MonitoredResourceType type) + { + String tblName = null; + switch (type) + { + case APACHE_RESOURCE: + tblName = __APACHE_RESOURCE_ACCESS_TABLE_NAME; + break; + case FRONT_RESOURCE: + tblName = __FRONT_RESOURCE_ACCESS_TABLE_NAME; + break; + case BACK_PAGE_RESOURCE: + tblName = __PAGE_RESOURCE_ACCESS_TABLE_NAME; + break; + case BACK_PAGE_ELEMENT: + tblName = __PAGE_ELEMENT_RESOURCE_ACCESS_TABLE_NAME; + break; + default: + throw new IllegalArgumentException("Illegal type. Allowed types are : " + Arrays.asList(MonitoredResourceType.values())); + } + + return tblName; + } + + /** + * Get resource access to be processed + * @param type + * @return The sql query + */ + public static String getResourceAccessToProcess(ResourceCacheStatsType type) + { + String colsFields = null; + String groupFields = null; + String internalClause = null; + switch (type) + { + case APACHE_ONLY: + colsFields = __APACHE_PROCESS_COLS_FIELDS; + groupFields = __APACHE_PROCESS_GROUP_FIELDS; + internalClause = __APACHE_INTERNAL_GET_RES_PATTERN; + break; + case FRONT_ONLY: + colsFields = __FRONT_PROCESS_COLS_FIELDS; + groupFields = __FRONT_PROCESS_GROUP_FIELDS; + internalClause = __FRONT_INTERNAL_GET_RES_PATTERN; + break; + case FRONT_FROM_APACHE: + colsFields = __FRONT_APACHE_PROCESS_COLS_FIELDS; + groupFields = __FRONT_APACHE_PROCESS_GROUP_FIELDS; + internalClause = __FRONT_APACHE_INTERNAL_GET_RES_PATTERN; + break; + case BACK_PAGE: + colsFields = __PAGE_PROCESS_COLS_FIELDS; + groupFields = __PAGE_PROCESS_GROUP_FIELDS; + internalClause = __PAGE_INTERNAL_GET_RES_PATTERN; + break; + case BACK_PAGE_ELEMENT: + colsFields = __PAGE_ELEMENT_PROCESS_COLS_FIELDS; + groupFields = __PAGE_ELEMENT_PROCESS_GROUP_FIELDS; + internalClause = __PAGE_ELEMENT_INTERNAL_GET_RES_PATTERN; + break; + default: + throw new IllegalArgumentException("Illegal type. Allowed types are : " + Arrays.asList(ResourceCacheStatsType.values())); + } + + String sql = StringUtils.replaceOnce(__GET_RESOURCE_TO_PROCESS, __TBL_NAME_PATTERN, getResourceAccessTableName(type)); + sql = StringUtils.replaceOnce(sql, __COLS_FIELDS_PATTERN, colsFields); + sql = StringUtils.replaceOnce(sql, __GROUP_FIELDS_PATTERN, groupFields); + sql = StringUtils.replaceOnce(sql, __INTERNAL_GET_RES_PATTERN, internalClause); + + return sql; + } + + /** + * Get the max id of the of the resources to process + * @param type + * @return The sql query + */ + public static String getMaxIdToProcessSql(ResourceCacheStatsType type) + { + switch (type) + { + case BACK_PAGE: + case BACK_PAGE_ELEMENT: + return StringUtils.replaceOnce(__GET_MAX_ID_TO_PROCESS, __TBL_NAME_PATTERN, getResourceAccessTableName(type)); + case APACHE_ONLY: + return StringUtils.replaceOnce(__GET_MAX_ID_TO_PROCESS_APACHE, __TBL_NAME_PATTERN, getResourceAccessTableName(type)); + case FRONT_ONLY: + return StringUtils.replaceOnce(__GET_MAX_ID_TO_PROCESS_FRONT, __TBL_NAME_PATTERN, getResourceAccessTableName(type)); + case FRONT_FROM_APACHE: + return StringUtils.replaceOnce(__GET_MAX_ID_TO_PROCESS_FRONT_APACHE, __TBL_NAME_PATTERN, getResourceAccessTableName(type)); + default: + throw new IllegalArgumentException("Illegal type. Allowed types are : " + Arrays.asList(ResourceCacheStatsType.values())); + } + } + + /** + * The mark as processed sql query + * @param type + * @return The sql query + */ + public static String getMarkProcessedSql(ResourceCacheStatsType type) + { + switch (type) + { + case BACK_PAGE: + case BACK_PAGE_ELEMENT: + return StringUtils.replaceOnce(__MARK_AS_PROCESSED, __TBL_NAME_PATTERN, getResourceAccessTableName(type)); + case APACHE_ONLY: + return StringUtils.replaceOnce(__MARK_AS_PROCESSED_APACHE, __TBL_NAME_PATTERN, getResourceAccessTableName(type)); + case FRONT_ONLY: + return StringUtils.replaceOnce(__MARK_AS_PROCESSED_FRONT, __TBL_NAME_PATTERN, getResourceAccessTableName(type)); + case FRONT_FROM_APACHE: + return StringUtils.replaceOnce(__MARK_AS_PROCESSED_FRONT_APACHE, __TBL_NAME_PATTERN, getResourceAccessTableName(type)); + default: + throw new IllegalArgumentException("Illegal type. Allowed types are : " + Arrays.asList(ResourceCacheStatsType.values())); + } + } + + /** + * The mark as processed sql query. + * This is a special case for the type FRONT_FROM_APACHE. + * After having marked as processed the front resources, the corresponding apache resource must be marked too. + * @return The sql query + */ + public static String getMarkApacheProcessedSql() + { + return __MARK_AS_PROCESSED_APACHE_AFTER_FRONT_APACHE; + } + + private static String getResourceAccessTableName(ResourceCacheStatsType type) + { + String tblName = null; + switch (type) + { + case APACHE_ONLY: + tblName = __APACHE_RESOURCE_ACCESS_TABLE_NAME; + break; + case FRONT_FROM_APACHE: + case FRONT_ONLY: + tblName = __FRONT_RESOURCE_ACCESS_TABLE_NAME; + break; + case BACK_PAGE: + tblName = __PAGE_RESOURCE_ACCESS_TABLE_NAME; + break; + case BACK_PAGE_ELEMENT: + tblName = __PAGE_ELEMENT_RESOURCE_ACCESS_TABLE_NAME; + break; + default: + throw new IllegalArgumentException("Illegal type. Allowed types are : " + Arrays.asList(ResourceCacheStatsType.values())); + } + + return tblName; + } + + /** + * The find sql cache stats entry query + * @param type + * @return The sql query + */ + public static String findCacheStatsEntrySql(ResourceCacheStatsType type) + { + String whereClause = null; + switch (type) + { + case APACHE_ONLY: + whereClause = __APACHE_WHERE_CLAUSE; + break; + case FRONT_ONLY: + whereClause = __FRONT_WHERE_CLAUSE; + break; + case FRONT_FROM_APACHE: + whereClause = __FRONT_APACHE_WHERE_CLAUSE; + break; + case BACK_PAGE: + whereClause = __PAGE_WHERE_CLAUSE; + break; + case BACK_PAGE_ELEMENT: + whereClause = __PAGE_ELEMENT_WHERE_CLAUSE; + break; + default: + throw new IllegalArgumentException("Illegal type. Allowed types are : " + Arrays.asList(ResourceCacheStatsType.values())); + } + + String sql = StringUtils.replaceOnce(__FIND_CACHE_STATS, __TBL_NAME_PATTERN, getCacheStatsTableName(type)); + sql = StringUtils.replaceOnce(sql, __WHERE_CLAUSE_PATTERN, whereClause); + + return sql; + } + + /** + * The insert sql cache stats entry query + * @param type + * @return The sql query + */ + public static String insertCacheStatsSql(ResourceCacheStatsType type) + { + String insertCols = null; + switch (type) + { + case APACHE_ONLY: + insertCols = __APACHE_INSERT_CACHE_COLS_CLAUSE; + break; + case FRONT_ONLY: + insertCols = __FRONT_INSERT_CACHE_COLS_CLAUSE; + break; + case FRONT_FROM_APACHE: + insertCols = __FRONT_APACHE_INSERT_CACHE_COLS_CLAUSE; + break; + case BACK_PAGE: + insertCols = __PAGE_INSERT_CACHE_COLS_CLAUSE; + break; + case BACK_PAGE_ELEMENT: + insertCols = __PAGE_ELEMENT_INSERT_CACHE_COLS_CLAUSE; + break; + default: + throw new IllegalArgumentException("Illegal type. Allowed types are : " + Arrays.asList(ResourceCacheStatsType.values())); + } + + String[] insertValuesArray = StringUtils.split(insertCols, ", "); + Arrays.fill(insertValuesArray, "?"); + String insertValues = StringUtils.join(insertValuesArray, ", "); + + String sql = StringUtils.replaceOnce(__INSERT_CACHE_STATS, __TBL_NAME_PATTERN, getCacheStatsTableName(type)); + sql = StringUtils.replaceOnce(sql, __INSERT_COLS_CLAUSE_PATTERN, insertCols); + sql = StringUtils.replaceOnce(sql, __INSERT_VALUES_CLAUSE_PATTERN, insertValues); + + return sql; + } + + /** + * The update sql cache stats entry query + * @param type + * @return The sql query + */ + public static String updateCacheStatsEntrySql(ResourceCacheStatsType type) + { + String setClause = null; + String whereClause = null; + switch (type) + { + case APACHE_ONLY: + setClause = __APACHE_SET_CLAUSE; + whereClause = __APACHE_WHERE_CLAUSE; + break; + case FRONT_ONLY: + setClause = __FRONT_SET_CLAUSE; + whereClause = __FRONT_WHERE_CLAUSE; + break; + case FRONT_FROM_APACHE: + setClause = __FRONT_APACHE_SET_CLAUSE; + whereClause = __FRONT_APACHE_WHERE_CLAUSE; + break; + case BACK_PAGE: + setClause = __PAGE_SET_CLAUSE; + whereClause = __PAGE_WHERE_CLAUSE; + break; + case BACK_PAGE_ELEMENT: + setClause = __PAGE_ELEMENT_SET_CLAUSE; + whereClause = __PAGE_ELEMENT_WHERE_CLAUSE; + break; + default: + throw new IllegalArgumentException("Illegal type. Allowed types are : " + Arrays.asList(ResourceCacheStatsType.values())); + } + + String sql = StringUtils.replaceOnce(__UPDATE_CACHE_STATS, __TBL_NAME_PATTERN, getCacheStatsTableName(type)); + sql = StringUtils.replaceOnce(sql, __SET_CLAUSE_PATTERN, setClause); + sql = StringUtils.replaceOnce(sql, __WHERE_CLAUSE_PATTERN, whereClause); + + return sql; + } + + private static String getCacheStatsTableName(ResourceCacheStatsType type) + { + String tblName = null; + switch (type) + { + case APACHE_ONLY: + case FRONT_ONLY: + case FRONT_FROM_APACHE: + tblName = __FRONT_CACHE_STATS_TABLE_NAME; + break; + case BACK_PAGE: + tblName = __PAGE_CACHE_STATS_TABLE_NAME; + break; + case BACK_PAGE_ELEMENT: + tblName = __PAGE_ELEMENT_CACHE_STATS_TABLE_NAME; + break; + default: + throw new IllegalArgumentException("Illegal type. Allowed types are : " + Arrays.asList(ResourceCacheStatsType.values())); + } + + return tblName; + } +} Index: main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/ApacheAccessLogImporter.java =================================================================== --- main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/ApacheAccessLogImporter.java (revision 0) +++ main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/ApacheAccessLogImporter.java (revision 0) @@ -0,0 +1,290 @@ +package org.ametys.plugins.site.cachemonitoring.core; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileReader; +import java.io.IOException; +import java.text.DateFormat; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Date; +import java.util.LinkedList; +import java.util.List; +import java.util.Locale; +import java.util.concurrent.Future; + +import org.apache.avalon.framework.activity.Disposable; +import org.apache.avalon.framework.component.Component; +import org.apache.avalon.framework.configuration.Configurable; +import org.apache.avalon.framework.configuration.Configuration; +import org.apache.avalon.framework.configuration.ConfigurationException; +import org.apache.avalon.framework.logger.AbstractLogEnabled; +import org.apache.avalon.framework.service.ServiceException; +import org.apache.avalon.framework.service.ServiceManager; +import org.apache.avalon.framework.service.Serviceable; +import org.apache.commons.io.FilenameUtils; +import org.joda.time.Duration; +import org.joda.time.format.PeriodFormat; + +import org.ametys.plugins.site.cachemonitoring.server.ApacheResourceAccess; +import org.ametys.runtime.config.Config; +import org.ametys.runtime.util.StringUtils; + + +/** + * Import Apache access log and pass them to the resource access monitor + */ +public class ApacheAccessLogImporter extends AbstractLogEnabled implements Component, Configurable, Serviceable, Disposable +{ + /** Avalon ROLE. */ + public static final String ROLE = ApacheAccessLogImporter.class.getName(); + + /** The resource access monitoring component */ + protected ResourceAccessMonitor _resourceAccessMonitor; + + /** + * Date of the initialization of the component, to ensure that only newer + * log entries are importer + */ + protected Date _initializationDate; + + private List _logFileImporters; + + @Override + public void service(ServiceManager manager) throws ServiceException + { + _resourceAccessMonitor = (ResourceAccessMonitor) manager.lookup(ResourceAccessMonitor.ROLE); + } + + @Override + public void configure(Configuration configuration) throws ConfigurationException + { + _initializationDate = new Date(); + _logFileImporters = new ArrayList(); + + String frontConfig = Config.getInstance().getValueAsString("front.cache.monitoring.apache.log.paths"); + Collection logFilePaths = StringUtils.stringToCollection(frontConfig); + + configureLogFiles(logFilePaths); + + initializeLogFileImporters(); + } + + @Override + public void dispose() + { + _initializationDate = null; + _logFileImporters = null; + } + + /** + * Scan the log files for each site importer. + */ + synchronized void scanLogFiles() + { + for (LogFileImporter importer : _logFileImporters) + { + importer.importEntries(); + } + } + + private void configureLogFiles(Collection paths) + { + for (String path : paths) + { + String dirLog = FilenameUtils.getFullPathNoEndSeparator(path); + String logFilename = FilenameUtils.getName(path); + File logFile = new File(dirLog, logFilename); + + _logFileImporters.add(new LogFileImporter(logFile)); + } + } + + /** + * Initialize the log file importers. This method must not be called during + * the initialization of the component. Instead the call should be delayed, + * and made during the first import attempt. + */ + protected void initializeLogFileImporters() + { + for (LogFileImporter importer : _logFileImporters) + { + importer.initialize(); + } + } + + /** + * A log file importer. + * This class is able to import logs from an access log file. + */ + protected final class LogFileImporter + { + private final File _file; + // cached date format per importer, avoid synchronization issues while staying efficient + private final DateFormat _df = new SimpleDateFormat("dd/MMM/yyyy:HH:mm:ss Z", Locale.US); + + private BufferedReader _br; + private boolean _initialized; + + /** + * Ctor + * @param logFile + */ + protected LogFileImporter(File logFile) + { + _file = logFile; + } + + /** + * Initialize the log file importer. + */ + public synchronized void initialize() + { + if (_initialized) + { + return; + } + + try + { + _br = new BufferedReader(new FileReader(_file)); + skipEntriesUntilEOS(); + + String msg = String.format("The log file importer for the file '%s' is now initialized", _file); + _logDebug(msg); + _initialized = true; + } + catch (IOException e) + { + _logError("Exception when initializing the LogFileImporter for the file : '" + _file + "'", e); + } + } + + /** + * Import new entries from the apache log file of this importer + */ + public synchronized void importEntries() + { + if (!_initialized) + { + _logError("LogFileImporter not initialized. Unable to import the new entries for the file : '" + _file + "'", null); + return; + } + + try + { + scanLogEntries(); + } + catch (IOException e) + { + _logError("IOException while importing apache access log.", e); + } + catch (Exception e) + { + _logError("Unexpected exception while importing apache access log.", e); + } + } + + /** + * Skip to the end of the buffered stream + * @throws IOException + */ + private void skipEntriesUntilEOS() throws IOException + { + long start = System.currentTimeMillis(); + + while (_br.readLine() != null) + { + // do nothing, just consume the buffered reader. + } + + String duration = PeriodFormat.getDefault().print(new Duration(System.currentTimeMillis() - start).toPeriod()); + String msg = String.format("It tooks %s to consume the whole log file '%s'", duration, _file); + _logDebug(msg); + } + + private void scanLogEntries() throws IOException + { + long start = System.currentTimeMillis(); + + List toAdd = new LinkedList(); + + // Scan the new lines of the log file line by line. + // For each line, create a new ApacheResourceAccess, and pass it to + // the resourceAccessMonitor component if this entry must be + // persisted. + boolean eosReached = false; + while (_br.ready() && !eosReached) + { + String entry = _br.readLine(); + if (entry != null) + { + ApacheResourceAccess r = ApacheResourceAccess.createRecord(entry, _df); + if (r != null) + { + if (r.isOfInterest(_initializationDate)) + { + toAdd.add(r); + } + else + { + _logDebug(String.format("This apache access log entry has been filtered out : %s", r)); + } + } + } + else + { + // End of stream has been reached + eosReached = true; + } + } + + Future future = _resourceAccessMonitor.addAccessRecords(toAdd); + + // Block until records have been consumed by the monitoring component + try + { + future.get(); + } + catch (Exception e) + { + _logError("Unexpected exception while waiting for the apache import task of the file '" + _file + "' to finish.", e); + } + + String durationStr = PeriodFormat.getDefault().print(new Duration(System.currentTimeMillis() - start).toPeriod()); + _logDebug(String.format("%s log entry(ies) scanned in %s", toAdd.size(), durationStr)); + } + } + + /** + * Helper log debug method + * @param msg The msg to log at the debug level. + */ + protected void _logDebug(String msg) + { + if (getLogger().isDebugEnabled()) + { + getLogger().debug(msg); + } + } + + /** + * Helper log error method + * @param msg The msg to log at the error level. + * @param t the throwable + */ + protected void _logError(String msg, Throwable t) + { + getLogger().error(msg, t); + } + + /** + * Helper log warn method + * @param msg The msg to log at the warn level. + */ + protected void _logWarn(String msg) + { + getLogger().warn(msg); + } +} Index: main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/ResourceCacheStats.java =================================================================== --- main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/ResourceCacheStats.java (revision 0) +++ main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/ResourceCacheStats.java (revision 0) @@ -0,0 +1,61 @@ +package org.ametys.plugins.site.cachemonitoring.core; + +import java.sql.PreparedStatement; +import java.sql.SQLException; + +/** + * A ResourceCacheStats is an object that holds cache statistics coming from the + * monitoring database or to be inserted/updated in the monitoring database. + */ +public interface ResourceCacheStats +{ + /** Type of possible ResourceCacheStats */ + public enum ResourceCacheStatsType + { + /** APACHE resource with a cache hit*/ + APACHE_ONLY, + /** Front resource with a unique id (coming from apache) */ + FRONT_FROM_APACHE, + /** Front resource without a unique id (direct to tomcat) */ + FRONT_ONLY, + /*** Back page resource */ + BACK_PAGE, + /*** Back page element resource stats */ + BACK_PAGE_ELEMENT; + } + + /** + * get type + * @return the type of this ResourceCacheStats instance. + */ + public ResourceCacheStatsType getType(); + + /** + * Must configure the given PreparedStatement for a find query. + * @param ps + * @throws SQLException + */ + public void configureSqlFind(PreparedStatement ps) throws SQLException; + + /** + * Configure the {@link PreparedStatement} used when doing the SQL insert + * query. + * @param ps The {@link PreparedStatement} + * @throws SQLException + */ + public void configureSqlInsert(PreparedStatement ps) throws SQLException; + + /** + * Configure the {@link PreparedStatement} used when doing the SQL update + * query. + * @param ps The {@link PreparedStatement} + * @throws SQLException + */ + public void configureSqlUpdate(PreparedStatement ps) throws SQLException; + + /** + * Returns the number of hits + * @return int the hits + */ + public int getHits(); +} Index: main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/ResourceAccess.java =================================================================== --- main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/ResourceAccess.java (revision 0) +++ main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/ResourceAccess.java (revision 0) @@ -0,0 +1,42 @@ +package org.ametys.plugins.site.cachemonitoring.core; + +import java.sql.PreparedStatement; +import java.sql.SQLException; + +/** + * Monitored resources. Each access to theses resources is monitored in order to + * be able to analyze cache efficiency and to calculate meaningful statistics + */ +public interface ResourceAccess +{ + /** Type of monitored resources */ + public enum MonitoredResourceType + { + /** APACHE resource */ + APACHE_RESOURCE, + /** Front resource */ + FRONT_RESOURCE, + /*** Back page resource */ + BACK_PAGE_RESOURCE, + /*** Back page element resource */ + BACK_PAGE_ELEMENT; + } + + /** + * get type + * + * @return the type of this {@link ResourceAccess} instance. + */ + public MonitoredResourceType getType(); + + /** + * Configure the {@link PreparedStatement} used when doing the SQL insert + * query. This method must be consistent with the {@link #configureSqlInsert(PreparedStatement)} + * method. + * + * @param ps The {@link PreparedStatement} + * @throws SQLException + */ + public void configureSqlInsert(PreparedStatement ps) throws SQLException; +} + Index: main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/ResourceAccessMonitor.java =================================================================== --- main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/ResourceAccessMonitor.java (revision 0) +++ main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/ResourceAccessMonitor.java (revision 0) @@ -0,0 +1,250 @@ +package org.ametys.plugins.site.cachemonitoring.core; + +import java.sql.BatchUpdateException; +import java.sql.Connection; +import java.sql.PreparedStatement; +import java.sql.SQLException; +import java.sql.Statement; +import java.util.HashMap; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.Future; + +import org.apache.avalon.framework.activity.Initializable; +import org.apache.avalon.framework.component.Component; +import org.apache.avalon.framework.logger.AbstractLogEnabled; +import org.joda.time.Duration; +import org.joda.time.format.PeriodFormat; + +import org.ametys.plugins.site.cachemonitoring.core.ResourceAccess.MonitoredResourceType; +import org.ametys.runtime.datasource.ConnectionHelper; + +import com.google.common.collect.LinkedListMultimap; +import com.google.common.collect.ListMultimap; +import com.google.common.collect.Multimaps; + +/** + * The RessourceAccessMonitor collects the resources that have been requested, + * and export them into a database. + */ +public class ResourceAccessMonitor extends AbstractLogEnabled implements Initializable, Component +{ + /** Avalon ROLE. */ + public static final String ROLE = ResourceAccessMonitor.class.getName(); + + /** + * List of pending {@link ResourceAccess} waiting to be exported to the + * database. + */ + protected ListMultimap _pendingRecords; + + private final ExecutorService _importerPool = Executors.newCachedThreadPool(); + + @Override + public void initialize() throws Exception + { + _pendingRecords = Multimaps.synchronizedListMultimap(LinkedListMultimap.create()); + } + + void exportPendings() + { + _logDebug("Start to insert pending records."); + + long start = System.currentTimeMillis(); + int totalCount = 0; + + Map> toExport = new HashMap>(); + synchronized (_pendingRecords) + { + totalCount = _pendingRecords.size(); + for (MonitoredResourceType mrt : _pendingRecords.keySet()) + { + toExport.put(mrt, _pendingRecords.removeAll(mrt)); + } + } + + int successCount = 0; + if (!toExport.isEmpty()) + { + Connection conn = null; + try + { + conn = ConnectionHelper.getConnection(CacheMonitoringSqlStatements.MONITORING_DATASOURCE_ID); + conn.setAutoCommit(false); + successCount = export(conn, toExport); + conn.commit(); + } + catch (Exception e) + { + _logError("Exception during export to DB. Performing a rollback of the transaction.", e); + + try + { + if (conn != null) + { + conn.rollback(); + } + } + catch (SQLException sqle) + { + _logError("Error : SQLException. Problem while performing the rollback.", sqle); + } + } + finally + { + ConnectionHelper.cleanup(conn); + } + } + + String durationStr = PeriodFormat.getDefault().print(new Duration(System.currentTimeMillis() - start).toPeriod()); + _logDebug(String.format("%s/%s pending records exported into db in %s", successCount, totalCount, durationStr)); + } + + private int export(Connection conn, Map> toExport) + { + try + { + int success = 0; + for (MonitoredResourceType mrt : toExport.keySet()) + { + success += exportByType(conn, mrt, toExport.get(mrt).iterator()); + } + return success; + } + catch (SQLException e) + { + e.printStackTrace(); + } + + return 0; + } + + private int exportByType(Connection conn, MonitoredResourceType mrt, Iterator rait) throws SQLException + { + PreparedStatement ps = null; + ResourceAccess ra = null; + try + { + String insertSql = CacheMonitoringSqlStatements.insertResourceAccessSql(mrt); + ps = conn.prepareStatement(insertSql, Statement.NO_GENERATED_KEYS); + + while (rait.hasNext()) + { + ra = rait.next(); + ra.configureSqlInsert(ps); + ps.addBatch(); + } + + try + { + return ps.executeBatch().length; + } + catch (BatchUpdateException e) + { + // more detailed treatment is possible through e.getUpdateCounts() + _logError("Batch exception while inserting new records into of type '" + mrt + "' to the DB", e); + } + } + finally + { + ConnectionHelper.cleanup(ps); + } + + return 0; + } + + /** + * Add a new {@link ResourceAccess} to the monitored resources. + * + * @param ra The resource access object. + * @return a {@link Future} to use if you would like to immediately block + * waiting for the add operation to finish. + */ + public Future addAccessRecord(ResourceAccess ra) + { + return _importerPool.submit(new RecordAdder(ra)); + } + + /** + * Add a new {@link ResourceAccess} to the monitored resources. + * + * @param lra The list of resource access objects. + * @return a {@link Future} to use if you would like to immediately block + * waiting for the add operation to finish. + */ + public Future addAccessRecords(List lra) + { + return _importerPool.submit(new RecordsAdder(lra)); + } + + /** + * {@link Callable} that add a {@link ResourceAccess} in the list of pending + * records. + */ + private class RecordAdder implements Callable + { + private final ResourceAccess _ra; + + public RecordAdder(ResourceAccess ra) + { + _ra = ra; + } + + @Override + public Void call() throws Exception + { + _pendingRecords.put(_ra.getType(), _ra); + return null; + } + } + + /** + * {@link Callable} that add each element a list of {@link ResourceAccess} into the + * list of pending records. + */ + private class RecordsAdder implements Callable + { + private final List _lra; + + public RecordsAdder(List lra) + { + _lra = lra; + } + + @Override + public Void call() throws Exception + { + for (ResourceAccess ra : _lra) + { + _pendingRecords.put(ra.getType(), ra); + } + return null; + } + } + + /** + * Helper log debug method + * @param msg The msg to log at the debug level. + */ + protected void _logDebug(String msg) + { + if (getLogger().isDebugEnabled()) + { + getLogger().debug(msg); + } + } + + /** + * Helper log error method + * @param msg The msg to log at the error level. + * @param t the throwable + */ + protected void _logError(String msg, Throwable t) + { + getLogger().error(msg, t); + } +} Index: main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/FrontCacheMonitoringScheduler.java =================================================================== --- main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/FrontCacheMonitoringScheduler.java (revision 0) +++ main/plugin-site/src/org/ametys/plugins/site/cachemonitoring/core/FrontCacheMonitoringScheduler.java (revision 0) @@ -0,0 +1,104 @@ +package org.ametys.plugins.site.cachemonitoring.core; + +import java.util.Calendar; +import java.util.GregorianCalendar; +import java.util.Timer; +import java.util.TimerTask; + +import org.apache.avalon.framework.activity.Disposable; +import org.apache.avalon.framework.activity.Initializable; +import org.apache.avalon.framework.logger.LogEnabled; +import org.apache.avalon.framework.logger.Logger; +import org.apache.avalon.framework.service.ServiceException; +import org.apache.avalon.framework.service.ServiceManager; +import org.apache.avalon.framework.service.Serviceable; +import org.joda.time.Duration; +import org.joda.time.format.PeriodFormat; + +/** + * Component responsible of the import of Apache access log file and the + * monitoring of the access of the resources. + */ +public class FrontCacheMonitoringScheduler extends TimerTask implements Initializable, Serviceable, Disposable, LogEnabled +{ + /** Logger */ + protected Logger _logger; + + /** Timer. */ + protected Timer _timer; + + /** Apache access logs importer */ + protected ApacheAccessLogImporter _apacheLogImporter; + + /** Resource Access Monitor */ + protected ResourceAccessMonitor _resourceAccessMonitor; + + @Override + public void enableLogging(Logger logger) + { + _logger = logger; + } + + @Override + public void initialize() throws Exception + { + if (_logger.isDebugEnabled()) + { + _logger.debug("Scheduling the FrontCacheMonitoringScheduler component to run its task each hours"); + } + + // Daemon thread + _timer = new Timer("FrontCacheMonitoringScheduler", true); + + // The task is executed each hour, starting at the next hour if not too late. + GregorianCalendar calendar = new GregorianCalendar(); + int minute = calendar.get(Calendar.MINUTE); + + if (minute > 55) + { + calendar.add(Calendar.HOUR_OF_DAY, 1); + } + + calendar.set(Calendar.MINUTE, 0); + calendar.set(Calendar.SECOND, 0); + calendar.set(Calendar.MILLISECOND, 0); + + _timer.scheduleAtFixedRate(this, calendar.getTime(), 60 * 60 * 1000); + } + + @Override + public void service(ServiceManager manager) throws ServiceException + { + _apacheLogImporter = (ApacheAccessLogImporter) manager.lookup(ApacheAccessLogImporter.ROLE); + _resourceAccessMonitor = (ResourceAccessMonitor) manager.lookup(ResourceAccessMonitor.ROLE); + } + + @Override + public void dispose() + { + cancel(); + _timer.cancel(); + _logger = null; + } + + @Override + public void run() + { + _logger.info("Starting the front cache monitoring process."); + + // Import apache logs. + long start = System.currentTimeMillis(); + _apacheLogImporter.scanLogFiles(); + long endImport = System.currentTimeMillis(); + + // Then export pending records. + _resourceAccessMonitor.exportPendings(); + long end = System.currentTimeMillis(); + + String totalDuration = PeriodFormat.getDefault().print(new Duration(end - start).toPeriod()); + String importDuration = PeriodFormat.getDefault().print(new Duration(endImport - start).toPeriod()); + String exportDuration = PeriodFormat.getDefault().print(new Duration(end - endImport).toPeriod()); + + _logger.info(String.format("\tThe whole front cache monitoring process took %s [apache log import : %s, export to db: %s]", totalDuration, importDuration, exportDuration)); + } +} Index: main/workspace-site/src/org/ametys/site/IsPageCacheableAction.java =================================================================== --- main/workspace-site/src/org/ametys/site/IsPageCacheableAction.java (revision 20303) +++ main/workspace-site/src/org/ametys/site/IsPageCacheableAction.java (working copy) @@ -32,6 +32,8 @@ import org.apache.excalibur.source.TraversableSource; import org.apache.excalibur.source.impl.FileSource; +import org.ametys.plugins.site.cachemonitoring.core.ResourceAccessMonitor; +import org.ametys.plugins.site.cachemonitoring.front.FrontResourceAccess; import org.ametys.runtime.util.LoggerFactory; /** @@ -42,16 +44,20 @@ /** The cache access component. */ protected CacheAccessManager _cacheAccess; + /** The resource access monitoring component */ + protected ResourceAccessMonitor _resourceAccessMonitor; + /** The source resolver. */ protected SourceResolver _resolver; - private Logger _logger = LoggerFactory.getLoggerFor("site.cache.log"); + private final Logger _logger = LoggerFactory.getLoggerFor("site.cache.log"); @Override public void service(ServiceManager serviceManager) throws ServiceException { super.service(serviceManager); _cacheAccess = (CacheAccessManager) serviceManager.lookup(CacheAccessManager.ROLE); + _resourceAccessMonitor = (ResourceAccessMonitor) serviceManager.lookup(ResourceAccessMonitor.ROLE); } @Override @@ -62,35 +68,44 @@ // Block until known if the page is cacheable. boolean cacheable = _cacheAccess.isCacheable(source); - String resourceURI = parameters.getParameter("uri", ""); + Request request = ObjectModelHelper.getRequest(objectModel); if (_logger.isDebugEnabled()) { - // the Apache mod_unique_id set a request header "UNIQUE_ID" - // that we could use to track request in log files - Request request = ObjectModelHelper.getRequest(objectModel); String uniqueId = request.getHeader("UNIQUE_ID"); - if (uniqueId != null) { _logger.debug(uniqueId + " - cacheable : " + cacheable); } } + // Resource monitoring + FrontResourceAccess fra = (FrontResourceAccess) request.getAttribute("PAGE_ACCESS"); + fra.setCacheable(cacheable); + // If the page isn't cacheable, unlock and generate (return null). + String resourceURI = parameters.getParameter("uri", ""); if (!cacheable) { + _resourceAccessMonitor.addAccessRecord(fra); + _cacheAccess.unlock(source); return null; } // If cacheable and the resource exists, we don't have to generate it. else if (cacheable && _resourceExists(resolver, resourceURI)) { + fra.setCacheHit2(true); + _resourceAccessMonitor.addAccessRecord(fra); + _cacheAccess.unlock(source); return EMPTY_MAP; } // Else, return null: the page will be generated into the cache and unlocked when it's done. + fra.setCacheHit2(false); + _resourceAccessMonitor.addAccessRecord(fra); + return null; } catch (Exception e) @@ -111,7 +126,7 @@ boolean exists = false; Source source = null; - try + try { source = resolver.resolveURI(resourceURI); @@ -121,20 +136,20 @@ source = SiteCacheHelper.getHashedFileSource(resolver, (FileSource) source); } - if (source.exists() && (!(source instanceof TraversableSource) || !((TraversableSource) source).isCollection())) + if (source.exists() && (!(source instanceof TraversableSource) || !((TraversableSource) source).isCollection())) { exists = true; } - } - catch (SourceNotFoundException e) + } + catch (SourceNotFoundException e) { // Do not log - } - catch (Exception e) + } + catch (Exception e) { getLogger().warn("Exception resolving resource " + resourceURI, e); } - finally + finally { if (source != null) { Index: main/workspace-site/src/org/ametys/site/BackOfficeRequestHelper.java =================================================================== --- main/workspace-site/src/org/ametys/site/BackOfficeRequestHelper.java (revision 20303) +++ main/workspace-site/src/org/ametys/site/BackOfficeRequestHelper.java (working copy) @@ -99,7 +99,7 @@ } /** - * Build a HttpClient request object that will be sent to the back-office to query the page. + * Build a HttpClient request object that will be sent to the back-office to query the page. * @param objectModel the current object model. * @param page the wanted page path. * @return the HttpClient request, to be sent to the back-office. @@ -193,7 +193,7 @@ return boRequest; } - + /** * Get the site's base server path. * @param url the site url object. @@ -222,7 +222,7 @@ return fullUri.toString(); } - + /** * Get the front-office request's parameters as an HttpClient MultipartEntity, * to be added to a POST back-office request. @@ -312,20 +312,27 @@ boRequest.addHeader("X-Ametys-FO-Login", user); } + // Add apache unique-id + String uuid = (String) request.getAttribute("Monitoring-UUID"); + if (uuid != null) + { + boRequest.addHeader("X-Ametys-FO-UUID", uuid); + } + // Forwarding headers - Enumeration headers = request.getHeaderNames(); + Enumeration headers = request.getHeaderNames(); while (headers.hasMoreElements()) { String headerName = headers.nextElement(); if (__AUTHORIZED_HEADERS.matcher(headerName).matches()) { // forward - Enumeration headerValues = request.getHeaders(headerName); + Enumeration headerValues = request.getHeaders(headerName); while (headerValues.hasMoreElements()) { String headerValue = headerValues.nextElement(); boRequest.addHeader(headerName, headerValue); - } + } } } } @@ -415,7 +422,7 @@ { // the wrapped entity's getContent() decides about repeatability InputStream wrappedin = wrappedEntity.getContent(); - + return new GZIPInputStream(wrappedin); } Index: main/workspace-site/src/org/ametys/site/ResourceExistsAction.java =================================================================== --- main/workspace-site/src/org/ametys/site/ResourceExistsAction.java (revision 20303) +++ main/workspace-site/src/org/ametys/site/ResourceExistsAction.java (working copy) @@ -17,33 +17,67 @@ package org.ametys.site; import java.util.Map; +import java.util.UUID; import org.apache.avalon.framework.logger.Logger; import org.apache.avalon.framework.parameters.Parameters; +import org.apache.avalon.framework.service.ServiceException; +import org.apache.avalon.framework.service.ServiceManager; import org.apache.avalon.framework.thread.ThreadSafe; import org.apache.cocoon.acting.ServiceableAction; +import org.apache.cocoon.environment.ObjectModelHelper; import org.apache.cocoon.environment.Redirector; +import org.apache.cocoon.environment.Request; import org.apache.cocoon.environment.SourceResolver; +import org.apache.commons.lang.StringUtils; import org.apache.excalibur.source.Source; import org.apache.excalibur.source.SourceNotFoundException; import org.apache.excalibur.source.TraversableSource; import org.apache.excalibur.source.impl.FileSource; +import org.ametys.plugins.site.cachemonitoring.core.ResourceAccessMonitor; +import org.ametys.plugins.site.cachemonitoring.front.FrontResourceAccess; +import org.ametys.runtime.config.Config; import org.ametys.runtime.util.LoggerFactory; /** * The cocoon resource exists action but that returns false for folders */ -public class ResourceExistsAction extends ServiceableAction implements ThreadSafe +public class ResourceExistsAction extends ServiceableAction implements ThreadSafe { + /** The resource access monitoring component */ + protected ResourceAccessMonitor _resourceAccessMonitor; + + // FIXME static final logger? private Logger _logger = LoggerFactory.getLoggerFor("site.cache.log"); @Override + public void service(ServiceManager serviceManager) throws ServiceException + { + super.service(serviceManager); + _resourceAccessMonitor = (ResourceAccessMonitor) serviceManager.lookup(ResourceAccessMonitor.ROLE); + } + + @Override public Map act(Redirector redirector, SourceResolver resolver, Map objectModel, String src, Parameters parameters) throws Exception { String resourceURI = parameters.getParameter("url", src); Source source = null; - try + + // Resource monitoring + // the Apache mod_unique_id set a request header "UNIQUE_ID" + // that we could use to track request in log files + Request request = ObjectModelHelper.getRequest(objectModel); + String uniqueId = request.getHeader("UNIQUE_ID"); + + // Set the random uuid as a request attribute to be added later in the possible requests to the BO. + String uuid = UUID.randomUUID().toString(); + request.setAttribute("Monitoring-UUID", uuid); + + String path = StringUtils.removeStart(resourceURI, Config.getInstance().getValueAsString("org.ametys.site.root")); + FrontResourceAccess fra = new FrontResourceAccess(uniqueId, uuid, parameters.getParameter("site", null), path); + + try { source = resolver.resolveURI(resourceURI); @@ -53,31 +87,40 @@ source = SiteCacheHelper.getHashedFileSource(resolver, (FileSource) source); } - if (source.exists() - && (!(source instanceof TraversableSource) || !((TraversableSource) source).isCollection())) + if (source.exists() + && (!(source instanceof TraversableSource) || !((TraversableSource) source).isCollection())) { if (_logger.isDebugEnabled()) { _logger.debug("Find resource '" + resourceURI + " in cache"); } + + fra.setCacheable(true); + fra.setCacheHit1(true); + _resourceAccessMonitor.addAccessRecord(fra); + return EMPTY_MAP; } - } - catch (SourceNotFoundException e) + } + catch (SourceNotFoundException e) { // Do not log } - catch (Exception e) + catch (Exception e) { getLogger().warn("Exception resolving resource " + resourceURI, e); - } - finally + } + finally { - if (source != null) + if (source != null) { resolver.release(source); } } + + fra.setCacheHit1(false); + request.setAttribute("PAGE_ACCESS", fra); + return null; } Index: main/workspace-site/sitemap.xmap =================================================================== --- main/workspace-site/sitemap.xmap (revision 20303) +++ main/workspace-site/sitemap.xmap (working copy) @@ -151,12 +151,14 @@ + +