=== modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java 2014-04-14 08:42:46 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/data/DefaultQueryPlanner.java 2014-04-19 10:51:12 +0000 @@ -47,6 +47,7 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; +import java.util.Set; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -55,6 +56,7 @@ import org.hisp.dhis.analytics.DataQueryParams; import org.hisp.dhis.analytics.Partitions; import org.hisp.dhis.analytics.QueryPlanner; +import org.hisp.dhis.analytics.partition.PartitionManager; import org.hisp.dhis.analytics.table.PartitionUtils; import org.hisp.dhis.common.BaseDimensionalObject; import org.hisp.dhis.common.DimensionType; @@ -79,15 +81,18 @@ { private static final Log log = LogFactory.getLog( DefaultQueryPlanner.class ); - //TODO shortcut group by methods when only 1 option? - @Autowired private OrganisationUnitService organisationUnitService; + @Autowired + private PartitionManager partitionManager; + // ------------------------------------------------------------------------- // DefaultQueryPlanner implementation // ------------------------------------------------------------------------- + //TODO shortcut group by methods when only 1 option? + public void validate( DataQueryParams params ) throws IllegalQueryException { @@ -334,6 +339,8 @@ public List groupByPartition( DataQueryParams params, String tableName, String tableSuffix ) { + Set validPartitions = partitionManager.getAnalyticsPartitions(); + List queries = new ArrayList(); if ( params.isSkipPartitioning() ) @@ -343,22 +350,30 @@ } else if ( params.getPeriods() != null && !params.getPeriods().isEmpty() ) { - ListMap partitionPeriodMap = PartitionUtils.getPartitionPeriodMap( params.getPeriods(), tableName, tableSuffix ); + ListMap partitionPeriodMap = PartitionUtils.getPartitionPeriodMap( params.getPeriods(), tableName, tableSuffix, validPartitions ); for ( Partitions partitions : partitionPeriodMap.keySet() ) { + if ( partitions.hasAny() ) + { + DataQueryParams query = params.instance(); + query.setPeriods( partitionPeriodMap.get( partitions ) ); + query.setPartitions( partitions ); + queries.add( query ); + } + } + } + else if ( params.getFilterPeriods() != null && !params.getFilterPeriods().isEmpty() ) + { + Partitions partitions = PartitionUtils.getPartitions( params.getFilterPeriods(), tableName, tableSuffix, validPartitions ); + + if ( partitions.hasAny() ) + { DataQueryParams query = params.instance(); - query.setPeriods( partitionPeriodMap.get( partitions ) ); query.setPartitions( partitions ); queries.add( query ); } } - else if ( params.getFilterPeriods() != null && !params.getFilterPeriods().isEmpty() ) - { - DataQueryParams query = params.instance(); - query.setPartitions( PartitionUtils.getPartitions( params.getFilterPeriods(), tableName, tableSuffix ) ); - queries.add( query ); - } else { throw new IllegalQueryException( "Query does not contain any period dimension items" ); === added directory 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/partition' === added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/partition/JdbcPartitionManager.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/partition/JdbcPartitionManager.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/partition/JdbcPartitionManager.java 2014-04-19 10:51:12 +0000 @@ -0,0 +1,99 @@ +package org.hisp.dhis.analytics.partition; + +/* + * Copyright (c) 2004-2014, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import static org.hisp.dhis.analytics.AnalyticsTableManager.ANALYTICS_TABLE_NAME; +import static org.hisp.dhis.analytics.AnalyticsTableManager.EVENT_ANALYTICS_TABLE_NAME; + +import java.util.HashSet; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.jdbc.core.JdbcTemplate; + +/** + * @author Lars Helge Overland + */ +public class JdbcPartitionManager + implements PartitionManager +{ + private static final Log log = LogFactory.getLog( JdbcPartitionManager.class ); + + private Set ANALYTICS_PARTITIONS = null; + private Set ANALYTICS_EVENT_PARTITIONS = null; + + @Autowired + private JdbcTemplate jdbcTemplate; + + public Set getAnalyticsPartitions() + { + if ( ANALYTICS_PARTITIONS != null ) + { + return ANALYTICS_PARTITIONS; + } + + final String sql = + "select table_name from information_schema.tables " + + "where table_name like '" + ANALYTICS_TABLE_NAME + "%' " + + "and table_type = 'BASE TABLE'"; + + log.info( "Information schema analytics SQL: " + sql ); + + Set partitions = new HashSet( jdbcTemplate.queryForList( sql, String.class ) ); + ANALYTICS_PARTITIONS = partitions; + return partitions; + } + + public Set getEventAnalyticsPartitions() + { + if ( ANALYTICS_EVENT_PARTITIONS != null ) + { + return ANALYTICS_EVENT_PARTITIONS; + } + + final String sql = + "select table_name from information_schema.tables " + + "where table_name like '" + EVENT_ANALYTICS_TABLE_NAME + "%' " + + "and table_type = 'BASE TABLE'"; + + log.info( "Information schema event analytics SQL: " + sql ); + + Set partitions = new HashSet( jdbcTemplate.queryForList( sql, String.class ) ); + ANALYTICS_EVENT_PARTITIONS = partitions; + return partitions; + } + + public void clearCaches() + { + ANALYTICS_PARTITIONS = null; + ANALYTICS_EVENT_PARTITIONS = null; + } +} === added file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/partition/PartitionManager.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/partition/PartitionManager.java 1970-01-01 00:00:00 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/partition/PartitionManager.java 2014-04-19 10:51:12 +0000 @@ -0,0 +1,52 @@ +package org.hisp.dhis.analytics.partition; + +/* + * Copyright (c) 2004-2014, University of Oslo + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are met: + * Redistributions of source code must retain the above copyright notice, this + * list of conditions and the following disclaimer. + * + * Redistributions in binary form must reproduce the above copyright notice, + * this list of conditions and the following disclaimer in the documentation + * and/or other materials provided with the distribution. + * Neither the name of the HISP project nor the names of its contributors may + * be used to endorse or promote products derived from this software without + * specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED + * WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE + * DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR + * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES + * (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON + * ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS + * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +import java.util.Set; + +/** + * @author Lars Helge Overland + */ +public interface PartitionManager +{ + /** + * Returns a set of names of current analytics partitions. + */ + Set getAnalyticsPartitions(); + + /** + * Returns a set of names of current event analytics partitions. + */ + Set getEventAnalyticsPartitions(); + + /** + * Clears the partition name caches. + */ + void clearCaches(); +} === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java 2014-03-18 08:10:10 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/DefaultAnalyticsTableService.java 2014-04-19 10:51:12 +0000 @@ -41,6 +41,7 @@ import org.hisp.dhis.analytics.AnalyticsTable; import org.hisp.dhis.analytics.AnalyticsTableManager; import org.hisp.dhis.analytics.AnalyticsTableService; +import org.hisp.dhis.analytics.partition.PartitionManager; import org.hisp.dhis.common.IdentifiableObjectUtils; import org.hisp.dhis.dataelement.DataElementService; import org.hisp.dhis.organisationunit.OrganisationUnitService; @@ -81,6 +82,9 @@ private SqlViewService sqlViewService; @Autowired + private PartitionManager partitionManager; + + @Autowired private Notifier notifier; // ------------------------------------------------------------------------- @@ -137,6 +141,8 @@ swapTables( tables ); + partitionManager.clearCaches(); + clock.logTime( "Table update done" ); notifier.notify( taskId, "Table update done" ); } === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/PartitionUtils.java' --- dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/PartitionUtils.java 2014-04-19 08:06:02 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/java/org/hisp/dhis/analytics/table/PartitionUtils.java 2014-04-19 10:51:12 +0000 @@ -89,7 +89,7 @@ return partitions.prunePartitions( validPartitions ); } - public static Partitions getPartitions( List periods, String tablePrefix, String tableSuffix ) + public static Partitions getPartitions( List periods, String tablePrefix, String tableSuffix, Set validPartitions ) { Set partitions = new HashSet(); @@ -98,16 +98,16 @@ partitions.addAll( getPartitions( (Period) period, tablePrefix, tableSuffix, null ).getPartitions() ); } - return new Partitions( new ArrayList( partitions ) ); + return new Partitions( new ArrayList( partitions ) ).prunePartitions( validPartitions ); } - public static ListMap getPartitionPeriodMap( List periods, String tablePrefix, String tableSuffix ) + public static ListMap getPartitionPeriodMap( List periods, String tablePrefix, String tableSuffix, Set validPartitions ) { ListMap map = new ListMap(); for ( NameableObject period : periods ) { - map.putValue( getPartitions( (Period) period, tablePrefix, tableSuffix, null ), period ); + map.putValue( getPartitions( (Period) period, tablePrefix, tableSuffix, null ).prunePartitions( validPartitions ), period ); } return map; === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/beans.xml' --- dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/beans.xml 2014-04-14 08:42:46 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/main/resources/META-INF/dhis/beans.xml 2014-04-19 10:51:12 +0000 @@ -14,6 +14,8 @@ + + === modified file 'dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java' --- dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java 2014-04-19 08:06:02 +0000 +++ dhis-2/dhis-services/dhis-service-analytics/src/test/java/org/hisp/dhis/analytics/table/PartitionUtilsTest.java 2014-04-19 10:51:12 +0000 @@ -73,7 +73,21 @@ periods.add( createPeriod( "200105" ) ); periods.add( createPeriod( "200108" ) ); - assertEquals( new Partitions().add( TBL + "_2000" ).add( TBL + "_2001" ), PartitionUtils.getPartitions( periods, TBL, null ) ); + assertEquals( new Partitions().add( TBL + "_2000" ).add( TBL + "_2001" ), PartitionUtils.getPartitions( periods, TBL, null, null ) ); + } + + @Test + public void getGetPartitionsMultiplePeriodsPrune() + { + List periods = new ArrayList(); + periods.add( createPeriod( "200011" ) ); + periods.add( createPeriod( "200105" ) ); + periods.add( createPeriod( "200108" ) ); + + Set validPartitions = new HashSet(); + validPartitions.add( TBL + "_2000" ); + + assertEquals( new Partitions().add( TBL + "_2000" ), PartitionUtils.getPartitions( periods, TBL, null, validPartitions ) ); } @Test @@ -120,7 +134,7 @@ public void testGetTablePeriodMapA() { ListMap map = PartitionUtils.getPartitionPeriodMap( getList( - createPeriod( "2000S1" ), createPeriod( "2000S2" ), createPeriod( "2001S1" ), createPeriod( "2001S2" ), createPeriod( "2002S1" ) ), TBL, null ); + createPeriod( "2000S1" ), createPeriod( "2000S2" ), createPeriod( "2001S1" ), createPeriod( "2001S2" ), createPeriod( "2002S1" ) ), TBL, null, null ); assertEquals( 3, map.size() ); @@ -137,7 +151,7 @@ public void testGetTablePeriodMapB() { ListMap map = PartitionUtils.getPartitionPeriodMap( getList( - createPeriod( "2000April" ), createPeriod( "2000" ), createPeriod( "2001" ), createPeriod( "2001Oct" ), createPeriod( "2002Oct" ) ), TBL, null ); + createPeriod( "2000April" ), createPeriod( "2000" ), createPeriod( "2001" ), createPeriod( "2001Oct" ), createPeriod( "2002Oct" ) ), TBL, null, null ); assertEquals( 5, map.size() ); @@ -147,4 +161,23 @@ assertTrue( map.keySet().contains( new Partitions().add( TBL + "_2001" ).add( TBL + "_2002" ) ) ); assertTrue( map.keySet().contains( new Partitions().add( TBL + "_2002" ).add( TBL + "_2003" ) ) ); } + + @Test + public void testGetTablePeriodMapPrune() + { + Set validPartitions = new HashSet(); + validPartitions.add( TBL + "_2000" ); + validPartitions.add( TBL + "_2002" ); + validPartitions.add( TBL + "_2003" ); + + ListMap map = PartitionUtils.getPartitionPeriodMap( getList( + createPeriod( "2000April" ), createPeriod( "2000" ), createPeriod( "2001" ), createPeriod( "2001Oct" ), createPeriod( "2002Oct" ) ), TBL, null, validPartitions ); + + assertEquals( 4, map.size() ); + + assertTrue( map.keySet().contains( new Partitions().add( TBL + "_2000" ) ) ); + assertTrue( map.keySet().contains( new Partitions().add( TBL + "_2000" ) ) ); + assertTrue( map.keySet().contains( new Partitions().add( TBL + "_2002" ) ) ); + assertTrue( map.keySet().contains( new Partitions().add( TBL + "_2002" ).add( TBL + "_2003" ) ) ); + } } === modified file 'dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/cache/ClearCacheAction.java' --- dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/cache/ClearCacheAction.java 2014-03-18 08:10:10 +0000 +++ dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/java/org/hisp/dhis/dataadmin/action/cache/ClearCacheAction.java 2014-04-19 10:51:12 +0000 @@ -28,13 +28,14 @@ * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +import org.hisp.dhis.analytics.partition.PartitionManager; import org.hisp.dhis.cache.HibernateCacheManager; +import org.springframework.beans.factory.annotation.Autowired; import com.opensymphony.xwork2.Action; /** * @author Lars Helge Overland - * @version $Id$ */ public class ClearCacheAction implements Action @@ -43,13 +44,12 @@ // Dependencies // ------------------------------------------------------------------------- + @Autowired private HibernateCacheManager cacheManager; - public void setCacheManager( HibernateCacheManager cacheManager ) - { - this.cacheManager = cacheManager; - } - + @Autowired + private PartitionManager partitionManager; + // ------------------------------------------------------------------------- // Action implementation // ------------------------------------------------------------------------- @@ -57,6 +57,7 @@ public String execute() { cacheManager.clearCache(); + partitionManager.clearCaches(); return SUCCESS; } === modified file 'dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/META-INF/dhis/beans.xml' --- dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/META-INF/dhis/beans.xml 2014-04-16 15:03:22 +0000 +++ dhis-2/dhis-web/dhis-web-maintenance/dhis-web-maintenance-dataadmin/src/main/resources/META-INF/dhis/beans.xml 2014-04-19 10:51:12 +0000 @@ -27,7 +27,6 @@ -