Thursday, 3 December 2015

Mongo DB Crib Sheet

General

Using school as the database and student as the collection.

use school;  // Change to use the school database

show collections;  // Show the collections which are available for this database

db.......pretty();  // Use pretty to make sure the output is formatted well

db.student.drop();  // Drop the collection

db.student.stats();  // Get the collection stats

db.student.totalIndexSize();   // Available from the stats but a short cut.  It is important that the indexes fit into memory

Find

db.students.find();

db.students.findOne();

db.students.find({name:'bob'});

db.students.find({name:'bob'}).count();  // Return the number of matches for this find

db.students.find({age:{$gt:15}});  // Find students with age greater than 15

db.students.find({age:{$gt:15, $lt:18}});  // Find students with age greater than 15 and less than 18

db.students.find({name:{$regex : 'q'}, email : {$exists : true}});   // Find all the students whose name includes a 'q' and who have an email 

db.students.find({$or:[{age:15}, {age:16}]});    // Students with age 15 or 16

db.students.find({pets:{$in:['dog','cat']}});    // Students with a pet dog or cat

db.students.find({pets:{$all:['dog','cat']}});    // Students with a pet dog and cat

db.students.find({age:{$gt:15}}, {"_id":0, name:1, age:1});     // Projection which changes the data fields that are returned.

db.students.find( {_id:4}, {pets: {$slice:2}});    // Return the first two elements of the pet array only

db.students.find().sort({age:1});    // Sort by age ascending

db.students.find().limit(20);    // Return the first 20 records

var cursor = db.students.find();
cursor.next();                               // A cursor is returned if the find query is allocated to a variable

db.students.find({$text : {$search : 'bob fred'}});   // Do a search on a text index for all the documents containing 'bob' or 'fred' (case insensitive)

db.students.find({address:{$near: [x,y]}});    // Returns values based on a 2d index in increasing distance often used with a limit(n)

db.students.find({address:{
        $near: {
            $geometry : {
                type : "Point",
                coordinates : [long, lat]},
            $maxDistance : 2000
        }
    }
});    // Returns values based on a 2dspherical index in increasing distance often used with a limit(n).  Searches for Lat & Long with a max distance

Insert

db.students.insert({name: 'Fred', age:17, results:[78.8, 89.9]});

Delete / Remove

db.students.remove({});  // Removes all documents from the collection

db.students.remove({name:'Fred'});  // Removes all Fred entries

db.students.remove({name:'Fred'}, {justOne:true});   // Removes the first 'Fred' entry found

Update

db.country.update({pop:{$gt:2000000}}, {$set: {large_country:'true'}}, {multi:true});      // Update all countries which have a population greater than 2Mil to have a new flag saying large_country=true

db.country.update({_id:'Japan'},{pop='30000000', lang='Japanese'});      // All other entries in the document are removed.

db.country.update({_id:'Japan'}, {$set : {countryCode:'JP'}});    // Add the countryCode = 'JP' to the document with _id = 'Japan'

db.country.update({_id:'Japan'}, {$unset : {countryCode:1}});    // Remove the countryCode key from the document with _id = 'Japan'

db.country.update({_id:'Japan'}, {$push : { languages: "Japanese"}});   // Adds 'Japanese' to the list of languages for 'Japan'

db.country.update({_id:'Japan'}, {$pop : { languages: -1}});   // Removes the first item from the list of languages for 'Japan'

db.country.update({_id:'Japan'}, {$addToSet : { languages: "English"}});   // Adds 'English' to the list of languages if it doesn't exist already

db.country.update({_id:'Japan'}, {$pushAll : { languages: ["Japanese", "English"]}});   // Adds 'Japanese' and 'English' to the list of languages even if they exist already
db.country.update({_id:'GB'}, {_id:'GB', languages:'English', pop:50000000}, {upsert:true}); // Looks to update the object but will insert it if it can't be found

Indexes

Index creation is run in the foreground by default and will lock the collection that is being indexed so that no read or write request will be published.

db.student.getIndexes();

db.student.createIndex({name:1, class:-1});  (1 = ascending, -1 = descending)

db.student.createIndex({name:1}, {unique:true});  // Create a unique index

db.student.createIndex({name:1}, {unique:true, sparse:true});  // Create a unique index which is also sparse.  Used when there could be null / missing values from a document.

db.student.createIndex({name:1}, {background:true});   // Create this index in the background without locking the collection

db.student.createIndex({'name.result':1});  // Create an index into the array of result embedded documents

db.student.createIndex({'name':'text'});  // Create an index into the array of name embedded documents

db.student.createIndex({'location':'2d'});  // Create an index based on a 2d location where 'location' is an array of two coordinates

db.student.createIndex({'location':'2dsphere'});  // Create an index based on a 2d spherical location where 'location' is an array of two coordinates latitude and longitude.

db.student.dropIndex({name:1});

Explain

Use the explain option to see which indexes are being used and other data.

db.student.explain().find({name:1});     // Explain the information about this find request

db.student.explain("executionStats").find({name:1});

db.student.explain("allPlansExecution").find({name:1});

Aggregation

Use the aggregation frame work to do groupings, counts, sums, averages etc

$group
db.student.aggregate([{$group:{_id : "$surname", count:{$sum:1}}}]);  // Count the number of students with the same surname

db.student.aggregate([{$group:{_id : {name : "$name", surname : "$surname"}, exam_avg : {$avg:"$result"}}};   // Use an _id as a document to identify the student.  Then average the exam marks

db.products.aggregate([{$group:{_id: $manufacturer, categories:{$addToSet:"$category"}}}]); // Build an array of the categories which each manufacturer has.  This builds a set so doesn't create duplicates in the array

db.products.aggregate([{$group:{_id: $manufacturer, categories:{$push:"$category"}}}]); // Build an array of the categories which each manufacturer has. $push allows duplicates in the array.

db.products.aggregate([{$group:{_id: $manufacturer, categories:{$max:"$price"}}}]); // Find the max price of a manufacturer product.

db.products.aggregate([{$group:{_id: $manufacturer, categories:{$min:"$price"}}}]); // Find the min price of a manufacturer product.

db.students.aggregate([
    {$group:{_id:{class_id:"$class_id", student_id:"$student_id"}, student_avg : {$avg:"$score"}}},
    {$group:{_id:"$_id.class_id", class_avg: {$avg:"$student_avg"}}}]);  // Average each students marks in a class, then average all the single

db.student.aggregate([{$sort: {category : 1, price : 1}}, {$group:{ _id:"$category", cheapest :{$first: "$price"}}}]);  // Sort by category & price and return the first value = cheapest per category

$project
Allows you to 'reshape' a document eg add keys, remove keys.  To keep a key put $keyName:1 otherwise the key will be removed - with the exception of _id which needs to be explicitly excluded.

db.products.aggregate([{$project :{
        _id:0,
        'maker':{$toLower:$manufacturer}, // Create a new key with the Lower Case of the manufacturer
        'details':{category:"$category",
                       price:{$multiply : [$price, 10]}},   // Create new document of category and 10xprice
        item: $name      // Keep the name as a new key called 'item'
    }}]);

$match
db.student.aggregate([{$match:{category : "tablet"}}]);  // Match values - very similar to find()

$sort
db.student.aggregate([{$sort:{price: 1}}]);  // Sort by price - very similar to sort()

$skip
db.student.aggregate([{$sort:{price: 1}, $skip : 10}]);  // Skip the first 10 records - only use with a sort first

$limit
db.student.aggregate([{$sort:{price: 1}, $limit : 5 }]);  // Only return 5 records - only use with a sort first

$unwind
db.student.aggregate([{$unwind : "$classes"}]); // Unwind the classes that the students have done


Profiling

With profiling on the db.system collection gets profiling information about queries which take longer than the threshold specified.

db.getProfilingLevel();

db.getProfilingStatus();

db.setProfilingLevel(1, 4);  // 1 is the level and 4 is the threshold ms for queries

db.system.profile.find();    // Find queries which have been profiled because they took too long.

db.system.profile.find({millis:{$gt:1000}}).sort({ts:-1});  // Find all the queries that took longer than 1 second sorted in timestamp descending order

mongostat is useful and is similar to iostat on a linux box.

mongotop # // samples the database every # seconds and prints out an overview of where mongo is slow.

Replication

General
rs.status();
rs.conf();      // Show the current configuration of the replica set
rs.slaveOk();   // Allow queries on a secondary in a replica set
rs.help();   // Get help on the replica set commands

Create
Create a replication set called "rs1" with a particular data path and listening on a particular port.
mongod --replSet rs1 --logpath "1.log" --dbpath /data/rs1 --port 27017 --fork

config = { _id : "rs1" , members:[
                                   {_id : 0, host : "192.168.1.1.27017"},
                                   {_id : 1, host : "192.168.1.2.27017"},                                    ]}
rs.initiate(config);










Tuesday, 19 May 2015

Database Testing with H2

H2 is an in-memory database which can be really useful for running unit tests against.  It allows the JPA annotated classes (Entities) to be used to write to and read from a real database.

Schema Generation

Use the previous post on Schema Generation from JPA objects to make sure that a schema.sql file exists in the test resources directory.  This can be used to create the database tables for H2 to then use.

Pom Dependencies

There are a few dependencies that need to be included in the pom file if they aren't already.  These include JUnit, H2 and spring, as well as hibernate entity manager.

    <dependency>
        <groupId>junit</groupId>
        <artifactId>junit</artifactId>
        <version>4.12</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>com.h2database</groupId>
        <artifactId>h2</artifactId>
        <version>1.3.174</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>org.springframework</groupId>
        <artifactId>spring-test</artifactId>
        <version>${spring.version}</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>org.springframework</groupId>
        <artifactId>spring-context</artifactId>
        <version>${spring.version}</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>org.springframework</groupId>
        <artifactId>spring-orm</artifactId>
        <version>${spring.version}</version>
        <scope>test</scope>
    </dependency>
    <dependency>
        <groupId>org.hibernate</groupId>
        <artifactId>hibernate-entitymanager</artifactId>
        <version>${hibernate.version}</version>
        <scope>test</scope>
    </dependency>

Spring Context for H2

This spring context (domain-test-context.xml) defines the use of H2 as the data source and initialises the database with a createSchema.sql file and the schema.sql file generated above by the hibernate4 maven plugin.

<beans xmlns="http://www.springframework.org/schema/beans"
    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
    xmlns:jdbc="http://www.springframework.org/schema/jdbc"
    xmlns:tx="http://www.springframework.org/schema/tx"
    xsi:schemaLocation="  
        http://www.springframework.org/schema/beans       
        http://www.springframework.org/schema/beans/spring-beans-4.0.xsd  
        http://www.springframework.org/schema/tx       
        http://www.springframework.org/schema/tx/spring-tx-4.0.xsd  
        http://www.springframework.org/schema/jdbc   
        http://www.springframework.org/schema/jdbc/spring-jdbc-4.0.xsd" >

    
    <jdbc:embedded-database id="dataSource" type="H2" />
    
    <jdbc:initialize-database data-source="dataSource" ignore-failures="ALL">
        <jdbc:script location="createSchema.sql" />
        <jdbc:script location="schema.sql" />
        <!-- We could put other initialising data stuff in here -->
    </jdbc:initialize-database>
    
    <!-- This manages the entities and interactions with the database -->
    <bean id="entityManagerFactory" class="org.springframework.orm.jpa.LocalContainerEntityManagerFactoryBean">
        <property name="packagesToScan" value="com.my.classes.impl" />
        <property name="dataSource" ref="wiDataSource" />
        <property name="jpaVendorAdapter">
            <bean class="org.springframework.orm.jpa.vendor.HibernateJpaVendorAdapter">
                <property name="showSql" value="false" />
                <property name="databasePlatform" value="org.hibernate.dialect.H2Dialect" />
                <property name="generateDdl" value="false" />
            </bean>
        </property>
        <property name="jpaProperties">
            <props>
                <!-- General settings -->
                <prop key="hibernate.archive.autodetection">class, hbm</prop>
                <prop key="hibernate.current_session_context_class">jta</prop>
                <prop key="hibernate.show_sql">true</prop>
                <prop key="hibernate.format_sql">true</prop>
            </props>
        </property>
    </bean>
    
    <tx:annotation-driven transaction-manager="transactionManager" proxy-target-class="true" />
    
    <bean id="transactionManager" class="org.springframework.orm.jpa.JpaTransactionManager">
        <property name="entityManagerFactory" ref="entityManagerFactory" />
    </bean>
    
    <bean class="org.springframework.orm.jpa.support.PersistenceAnnotationBeanPostProcessor" />
       
    <bean id="myRepository" class="com.my.classes.repo.MyRepository" />
</beans>

This context creates the H2 database and wires the MyRepository ready for testing.

The createSchema.sql file contains the line as the schema.sql generated won't generate a database schema itself only the tables / columns which are part of that schema.

create schema MY_SCHEMA;

Test Class

This is the sample test class.  It uses the Spring Runner with the text context above.  It is ApplicationContextAware so that we can load the repository (@Autowire could be used instead) and trust spring to use the PersistenceAnnotationBeanPostProcessor to inject the EntityManager.

@RunWith(SpringJUnit4ClassRunner.class)
@ContextConfiguration(locations = {"classpath:domain-test-context.xml"})
public class WorkInstructionRepositoryTest implements ApplicationContextAware
{
    /**
     * This is the application context that we can load beans from.
     */
    private ApplicationContext applicationContext;

    /**
     * JPA Entity Manager.
     */
    @PersistenceContext
    private EntityManager entityManager;

    /**
     * The class under test.
     */
    private MyRepository repo;

    /**
     * {@inheritDoc}
     */
    @Override
    public void setApplicationContext(final ApplicationContext applicationContext) throws BeansException
    {
        this.applicationContext = applicationContext;
    }

    /**
     * Before each test get the repository.
     */
    @Before
    public void before()
    {
        this.repo = (MyRepository) applicationContext.getBean("myRepository");
    }

    /**
     * Test creating and loading some data.
     */
    @Test
    @Transactional
    @Rollback(true)
    public void testCreateAndReadData()
    {
        // Arrange
        final SomeData data = new SomeData();
        data.setActive(true);
        data.setText("Some Data Text");

        // Act
        repo.createData(data);
        final List<DataInterface> loadedDataObjs = repo.getAllData();

        // Assert
        final DataInterface loadedData = loadedDataObjs.get(0);
        assertEquals(data.isActive(), loadedData.isActive());
        assertEquals(data.getText(), loadedData.getText());
    }
}

JPA Schema Generator

This is a sample of using the de.juplo plugin for generating a database schema from annotated JPA Entity classes.

The Config


 <plugin>
    <groupId>de.juplo</groupId>
    <artifactId>hibernate4-maven-plugin</artifactId>
    <version>1.1.0</version>
    <executions>
        <execution>
            <id>package</id>
            <phase>prepare-package</phase>
            <goals>
                <goal>export</goal>
            </goals>
            <configuration>
                 <hibernateDialect>org.hibernate.dialect.MySQL5Dialect</hibernateDialect>
            </configuration>
        </execution>
        <execution>
            <id>test</id>
            <phase>process-test-resources</phase>
            <goals>
                <goal>export</goal>
            </goals>
            <configuration>
                <outputFile>${project.build.testOutputDirectory}/schema.sql</outputFile>
                <hibernateDialect>org.hibernate.dialect.H2Dialect</hibernateDialect>
            </configuration>
        </execution>
    </executions>
    <configuration>
        <target>SCRIPT</target>
    </configuration>
 </plugin>

Note that the normal configuration doesn't need an outputFile because it is generated in the target directory by default.  This example uses two executions, the first for the standard target build and the second for a test build for use with H2 for unit testing.
The <target>SCRIPT</target> part means that a sql script is created rather than trying to execute this directly on a database.

For more information see http://juplo.de/hibernate4-maven-plugin/

Wednesday, 11 March 2015

JavaFX Filter ComboBox

Quite a lot of posts exist on the web about how hard it is to do a filter / filtered / filtering combo box in JavaFX.  Using one of these as a basis here is an example of what to do.  The key part is creating a StringConverter so that the strings that are displayed in the ComboBox can be translated to and from the underlying objects.

FilterComboBox

This class is a extension of the standard JavaFX ComboBox. The list of initialItems needs to be populated either through the constructor or by using the setInitialItems() method so that there is a definitive list to go back to when necessary.

import java.util.ArrayList;

/**
 * A control which provides a filtered combo box.  As the user
 * enters values into the combo editor the list is filtered automatically.
 *
 * @param <T> the object type that is held by this FilteredComboBox
 */
public class FilterComboBox<T extends Object> extends ComboBox<T>
{
    /**
     * The default / initial list that is in the combo when nothing
     * is entered in the editor.
     */
    private Collection<T> initialList = new ArrayList<>();

    /**
     * Check type.  True if this is startsWith, false if it is contains.
     */
    private final boolean startsWithCheck;

    /**
     * Constructs a new FilterComboBox with the given parameters.
     *
     * @param startsWithCheck true if this is a 'startsWith' check false if it is 'contains' check
     */
    public FilterComboBox(final boolean startsWithCheck)
    {
        super();
        this.startsWithCheck = startsWithCheck;

        super.setEditable(true);

        this.configAutoFilterListener();
    }

    /**
     * Constructs a new FilterComboBox with the given parameters.
     *
     * @param items The initial items
     * @param startsWithCheck true if this is a 'startsWith' check false if it is 'contains' check
     */
    public FilterComboBox(final ObservableList<T> items, final boolean startsWithCheck)
    {
        super(items);
        this.startsWithCheck = startsWithCheck;
        super.setEditable(true);
        initialList = items;

        this.configAutoFilterListener();
    }

    /**
     * Set the initial list of items into this combo box.
     *
     * @param initial The initial list
     */
    public void setInitialItems(final Collection<T> initial)
    {
        super.getItems().clear();
        super.getItems().addAll(initial);
        this.initialList = initial;
    }

    /**
     * Set up the auto filter on the combo.
     */
    private void configAutoFilterListener()
    {
        this.getEditor().textProperty().addListener(new ChangeListener<String>()
        {
            @Override
            public void changed(final ObservableValue<? extends String> observable, final String oldValue, final String newValue)
            {
                final T selected = getSelectionModel().getSelectedItem();
                if (selected == null
                        || !getConverter().toString(selected).equals(getEditor().getText()))
                {
                    filterItems(newValue);

                    if (getItems().size() == 1)
                    {
                        setUserInputToOnlyOption();
                        hide();
                    }
                    else if (!getItems().isEmpty())
                    {
                        show();
                    }
                }
            }
        });
    }

    /**
     * Method to filter the items and update the combo.
     *
     * @param filter The filter string to use.
     */
    private void filterItems(final String filter)
    {
        final ObservableList<T> filteredList = FXCollections.observableArrayList();
        for (T item : initialList)
        {
            if (startsWithCheck && getConverter().toString(item).toLowerCase().startsWith(filter.toLowerCase()))
            {
                filteredList.add(item);
            }
            else if (!startsWithCheck && getConverter().toString(item).toLowerCase().contains(filter.toLowerCase()))
            {
                filteredList.add(item);
            }
        }

        setItems(filteredList);
    }

    /**
     * If there is only one item left in the combo then we assume this is correct.
     * Put the item into the editor but select the end of the string that the user
     * hasn't actually entered.
     */
    private void setUserInputToOnlyOption()
    {
        final String onlyOption = getConverter().toString(getItems().get(0));
        final String currentText = getEditor().getText();
        if (onlyOption.length() > currentText.length())
        {
            getEditor().setText(onlyOption);
            Platform.runLater(new Runnable()
            {
                @Override
                public void run()
                {
                    getEditor().selectAll();
                }
            });
        }
    }
}

StringConverter

The key to this working correctly is a StringConverter object which allows JavaFX to convert properly from the Object in the ComboBox to the String which is displayed in the ComboBox.  The StringConverter is a standard JavaFX object which has a toString() and fromString() method.  The easiest way to get this working is to construct a StringConverter and provide the same list of items to it that is provided to the ComboBox.


public class MyObjectStringConverter extends StringConverter<MyObject> 
{
    /** The list of objects to do the conversions with. */
    private List<MyObject> myObjList;

    /**
     * Construct this object with the list for converting.
     *
     * @param items The items list
     *
     */
    public MyStringConverter(List<MyObject> items)
    {
        this.myObjList = items;
    }

    @Override
    public String toString(final MyObject myObj)
    {
        if (myObj != null)
        {
            return myObj.getName();
        }
        return null;
    }

    @Override
    public MyObject fromString(final String item)
    {
        for (MyObject myObj : myObjList)
        {
            if (myObj.getName().equals(item))
            {
                return myObj;
            }
        }
        return null;
    }
}

Usage

The usage of this combo is extremely easy.  Other than the standard ComboBox options in JavaFX the only things really necessary are making sure the initialList is set through the constructor or directly and making sure that the converter is used.

    final List<MyObject> myObjs = new ArrayList<>();
    ...

    final MyObjectStringConverter converter = new MyObjectStringConverter(myObjs);

    final FilterComboBox comboBox = new FilterComboBox(true);
    comboBox.setInitialItems(myObjs);
    comboBox.setConverter(converter);