admin – Moazzam's Ramblings

March 19, 2018March 19, 2018

Go – Finding and removing duplicate files on a Mac

My laptop’s hard drive is plagued with the same family pics stored in different folders. So, I wrote a simple app in Go to take care of this for me. The app does this by comparing MD5 hashes of files.

Usage

Assuming you saved the code in a file called main.go, here is how you can use it to find duplicate files:

go run main.go -dir /some/dir -dir=/another/dir

1	go run main.go -dir /some/dir -dir=/another/dir

This will print duplicate files to the terminal. I added the flag -dupe_action=sym flag in the app which will sym-link all duplicates to one file.

go run main.go -dir /some/dir -dir=/another/dir -dupe_action=sym

1	go run main.go -dir /some/dir -dir=/another/dir -dupe_action=sym

package main

import (
	"os/exec"
	"fmt"
	"strings"
	"os"
	"crypto/md5"
	"io"
	"encoding/hex"
	"flag"
)

// This type is here so we can accept a list of directories
type arrayFlags []string

func (r *arrayFlags) String() string {
	return "something"
}

func (r *arrayFlags) Set(value string) error {
	*r = append(*r, value)
	return nil
}

// getFilesInDirectories gets all the files in directories recursively
func getFilesInDirectories(dir []string) []string {
	var ret []string

	for _, item := range dir {
		// This could have been done with pure Go, but I was lazy             
		c := exec.Command(`find`, item, `-type`, `f`, `-iname`, `*.jpg`)
		outBytes, _ := c.Output()

		files := strings.Split(string(outBytes), "\n")
		for _, fileItem := range files {
			ret = append(ret, fileItem)
		}
	}

	return ret
}

// getFileMd5 returns the md5 hash of a file
func getFileMd5(path string) (string, error) {
	file, err := os.Open(path)
	if err != nil {
		return ``, err
	}

	defer file.Close()

	hash := md5.New()
	if _, err := io.Copy(hash, file); err != nil {
		return ``, err
	}

	hashInBytes := hash.Sum(nil)[:16]
	ret := hex.EncodeToString(hashInBytes)

	return ret, nil
}

func main() {
	var dupeAction string
	var lookInDirectories arrayFlags

	flag.StringVar(&dupeAction, "action", ``, `Action to take with duplicate files. Value: sym`)
	flag.Var(&lookInDirectories, `dir`, `Directory to look in`)

	flag.Parse()

	// Get all relevant files
	output := getFilesInDirectories(lookInDirectories)

	fileMap := make(map[string][]string)
	for _, file := range output {
		hash, err := getFileMd5(file)
		if err != nil {
			fmt.Println("MD5_ERROR", file, err)
		}
		fileMap[hash] = append(fileMap[hash], file)
	}

	// Print dupes
	for _, item := range fileMap {
		// Do nothing if the file has no duplicates
		if len(item) <= 1 {
			continue
		}

		firstFile := item[0]
		for i := 1; i < len(item); i++ {
			file := item[i]
			fmt.Println(file)

			// Sym link dupes if flag was set
			if dupeAction == `sym` {
				err := os.Remove(file)
				if err != nil {
					fmt.Println(`Failed to delete `, file, err)
				}
				exec.Command(`ln`, `-s`, firstFile, file).Run()
			}
		}
		fmt.Println()
	}
}

100

101

102

103

104

105

106

107

108

package main

import (

"os/exec"

"fmt"

"strings"

"os"

"crypto/md5"

"io"

"encoding/hex"

"flag"

)

// This type is here so we can accept a list of directories

type arrayFlags []string

func (r *arrayFlags) String() string {

return "something"

}

func (r *arrayFlags) Set(value string) error {

*r = append(*r, value)

return nil

}

// getFilesInDirectories gets all the files in directories recursively

func getFilesInDirectories(dir []string) []string {

var ret []string

for _, item := range dir {

// This could have been done with pure Go, but I was lazy

c := exec.Command(`find`, item, `-type`, `f`, `-iname`, `*.jpg`)

outBytes, _ := c.Output()

files := strings.Split(string(outBytes), "\n")

for _, fileItem := range files {

ret = append(ret, fileItem)

}

return ret

}

// getFileMd5 returns the md5 hash of a file

func getFileMd5(path string) (string, error) {

file, err := os.Open(path)

if err != nil {

return ``, err

}

defer file.Close()

hash := md5.New()

if _, err := io.Copy(hash, file); err != nil {

return ``, err

}

hashInBytes := hash.Sum(nil)[:16]

ret := hex.EncodeToString(hashInBytes)

return ret, nil

}

func main() {

var dupeAction string

var lookInDirectories arrayFlags

flag.StringVar(&dupeAction, "action", ``, `Action to take with duplicate files. Value: sym`)

flag.Var(&lookInDirectories, `dir`, `Directory to look in`)

flag.Parse()

// Get all relevant files

output := getFilesInDirectories(lookInDirectories)

fileMap := make(map[string][]string)

for _, file := range output {

hash, err := getFileMd5(file)

if err != nil {

fmt.Println("MD5_ERROR", file, err)

}

fileMap[hash] = append(fileMap[hash], file)

}

// Print dupes

for _, item := range fileMap {

// Do nothing if the file has no duplicates

if len(item) <= 1 {

continue

}

firstFile := item[0]

for i := 1; i < len(item); i++ {

file := item[i]

fmt.Println(file)

// Sym link dupes if flag was set

if dupeAction == `sym` {

err := os.Remove(file)

if err != nil {

fmt.Println(`Failed to delete `, file, err)

}

exec.Command(`ln`, `-s`, firstFile, file).Run()

}

fmt.Println()

}

November 16, 2017November 16, 2017

Gogland build expired (renamed to Goland)

This confused me for a few minutes so I am writing this for those who have the same issue. Jetbrains renamed Gogland Goland. They also expired Gogland build. So you have to go to their site and download Goland and use that. Remove Gogland completely from your computer

May 18, 2017May 19, 2017

How are paints made for various painting mediums?

Paints are made by adding pigment (color) to something that will help it stick to the surface of a canvas (or anything else it can be painted on).

Color
Color, also referred to as pigment, is a powder with a specific color. For artist grade and premium grade paints this is ground up metal in most cases. Cobalt blue is made with Cobalt powder mixed into a medium.

For Student grade paints, the pigment is usually obtained by using a colored die and mixing it with chalk (the one you write on a black board with) or marble dust. This helps manufacturers reduce the cost of the paint.

Oil Paints
Oil paints are generally made by grinding the pigment powder with linseed oil. The grinding can be done by manually (with grinding stone, etc) or using a machine that can do it much faster.

Linseed oil has forms the most flexible film when it dries compared to other drying oils. The disadvantage of linseed oil is that it tends to yellow more than other oils over time. The other oil used for making oil paints is walnut oil, which doesn’t yellow as much but is forms a less flexible film than linseed oil does. It is, sometimes, mixed with linseed oil so the resultant paint will have a strong film and yellow less.

Cheaper oil paints tend to have less pigment and oil and more filler (marble dust, chalk or any other filler). Artists grade paints have less filler but they do contain some. Premium grade paints have no filler. They may, however, have stabilizers for certain pigments.

Acrylic paints
Acrylic paints are made by mixing pigment into acrylic emulsions (plastic). Like oil paints, artist grade acrylic paints have more pigment and than student grade paints.

Watercolor paints
The medium for watercolor paints is not water. It is gum arabic. Pigment added to gum arabic to make watercolors. Gum arabic is soluble in water which is why water can be used with watercolors as medium (to move the paint around, etc)

Egg Tempera
This is not as popular as the other 3 mediums but this has been around for a while. Egg tempera paints are made by mixing pigment into the egg yolk (not the egg white).

Encaustic paints
Encaustic paints are wax sticks that heated and the wax is used for painting while it is hot. When the wax dries it cannot be manipulated. Encausing paints are made by adding pigment to heated beeswax

September 26, 2015

Oil vs Acrylic – What’s easier to clean?

Acrylics are much easier to clean than oils (and cheaper).

Brushes

With acrylics, you can get away with using just water to clean your brushes (depending on how thick you laid your paint). Most of the times, this is true even for impasto techniques. For oils, you have to use turpentine (as oil paints don’t dissolve in water). It is, however, not enough to just use turpentine to clean your brushes.

With oils, you have to first, soak your brushes in turpentine, get most of the paint out, then use soap and water to make sure there is no oil reside in the brush. If you just use turpentine to clean the brush (especially your liners) then the brush will get stiff over time and you won’t be able to paint with it anymore. I learned this lesson the hard way.

Stephan Baumann, an artist who videos I watched on Youtube, recommends using turpentine, then soak brushes in “Awesome” cleaning liquid, then use soap and water. Apparently, turpentine will damage your pipes (even I didn’t know that).

Hands

Acrylics dry fast. So fast that it is sometimes annoying while painting. This, however, is very good for cleaning up. You can dry your hands with soap and water, scrape the paint off your hands, etc.

Oil paints take a while to dry out but they are not that much harder to get off your hands. You should be okay with soap and water, just make sure you do a thorough job of cleaning your hands. You don’t want to get cancer or any such diseases.

Pallettes

If you use glass palettes, then both of them are easy to clean. Just use a glass scraper and you are good to go. Although I have found acrylics to take less effort when removing paint off pallettes.

One issue I have had when painting smaller paintings (with less paint) is acrylics drying on the palette before I am done painting. So, I have had to use a make-shift stay-wet palette, which is use-and-throw. This means no cleaning at all. You could use the same sheets for oil painting too. So overall there is little to no difference between them.

I haven’t used wooden palettes so I can’t talk about them but removing acrylic paint from acrylic palettes is a nightmare. I have heard it is much easier to remove oil paints from it. So, in the end the ease of cleaning depends on the type of palette you use.

September 21, 2015

Sunlit Garden

I have been painting landscapes mostly so I wanted to try something different this time. Another reason I wanted to do this painting is to experiment with sunlight (and little details like the gate) change how the viewer feels when he/she looks at the painting. Thankfully, my experiment was successful. Here is the painting:

Sunlit Garden

I added the sun rays by mixing titanium white with fluid medium, then painting straight lines. After that, lightly brushing over the white lines in the direction of the light and blending it with the object it is falling on. If you use water to thin the paint, the paint dries up too fast to do any blending.

June 29, 2015October 3, 2015

PHPSpec: Mocking methods of the object being tested

PHPSpec is very opinionated and won’t let you mock anything of an object being tested. It helps you write better code. This, however, causes issues when you are writing specs for legacy code which (usually) was not well designed.

There is a way you can get around this limitation – a child class. Let’s say the class you want to test is called “Car” and it has a method called “getNumberOfDoors” that you need to mock. Just write a child class and mock the method in it. Here is the code:

// Car.php
class Car
{
    // We want to test this method
    public function isSportsCar()
    {
        return $this->getNumberOfDoors() <= 3;
    }

    // It would make life much easier if we can mock this method
    // to test isSportsCar() method above
    public function getNumberOfDoors()
    {
        return 4;
    }
}

// CarSpec.php

// ..namespace and use statements here .. 

class CarSpec extends ObjectBehavior
{
    public function let()
    {
        $this->beADoubleOf('CarSpy');
    }

    public function it_can_identify_car_type()
    {
        // And viola! we can test to our heart's content for all possible
        // edge cases

        $this->setNumberOfDoors(2);
        $this->isSportsCar()->shouldReturn(true);
    }
}

// The child class which we can use to manipulate the actual class
// we want to test
class CarSpy extends Car
{
    protected $numDoors;

    public function getNumberOfDoors()
    {
       return $this->numDoors;
    }

    public function setNumberOfDoors($val)
    {
        $this->numDoors = $val;
    }
}

// Car.php

class Car

{

// We want to test this method

public function isSportsCar()

{

return $this->getNumberOfDoors() <= 3;

}

// It would make life much easier if we can mock this method

// to test isSportsCar() method above

public function getNumberOfDoors()

{

return 4;

}

// CarSpec.php

// ..namespace and use statements here ..

class CarSpec extends ObjectBehavior

{

public function let()

{

$this->beADoubleOf('CarSpy');

}

public function it_can_identify_car_type()

{

// And viola! we can test to our heart's content for all possible

// edge cases

$this->setNumberOfDoors(2);

$this->isSportsCar()->shouldReturn(true);

}

// The child class which we can use to manipulate the actual class

// we want to test

class CarSpy extends Car

{

protected $numDoors;

public function getNumberOfDoors()

{

return $this->numDoors;

}

public function setNumberOfDoors($val)

{

$this->numDoors = $val;

}

March 4, 2013

A second look at MariaDB

After my last post on MariaDB, I installed it and played around with it. It really is a project that was forked from MySQL some time ago since MySQL is increasingly becoming closed source. There is another fork of MySQL called Drizzle, which claims that they have made the database faster by removing chunks from the code that were not relevant to web related queries.

The reason I was very excited about MariaDB is that it claims to be a hybrid of RDBMS and noSQL philosophies. However, their stable version (5.5 as of this post) lacks proper support for noSQL queries. It supports noSQL through something people at MariaDB call “dynamic columns”. The dynamic columns work as intended if you use numeric keys for your column names. However, if you use alphabetical keys, you can have just one column. If you try to add multiple keys, it will error out. If you try to add more keys using an update statement, it will replace the key you had before.

All hope is not lost, though. They do have a version (10.0.1 as of this post) which worked properly in my tests but it is in alpha stage and it will be some time before it becomes stable. If you are not looking for a noSQL solution the latest stable version is still great. It has percona’s proprietary engine (Xtradb) which is a better implementation of InnoDB, a better MyISAM engine, etc.

February 3, 2012March 1, 2012

Zend Server and Zend Studio9

If you have Zend Server installed and you have an instance of MySQL that didn’t come with Zend Server, then Zend Studio will try to connect to the one that came with Zend Server. I discovered this the hard way when my PHPUnit tests kept failing in Zend Studio but ran perfectly from the command line (I had PHPUnit installed from PEAR and the one that came with Zend Studio).

So I decided to make Zend Server connect to my default MySQL instance instead of uninstalling Zend Server (because I want to play with it). For those with the same problem, you can do this:

# Rename Zend Studio's socket file to something else
# If you didn't choose the default, installation path then 
# use the path to that location instead
mv /usr/local/zend/mysql/tmp/mysql.sock /usr/local/zend/mysql/tmp/mysql_old.sock

# My MySQL installation's socket file is located in /tmp
# If your's is different, then you will have to modify the source path

ln -s /tmp/mysql.sock /usr/local/zend/mysql/tmp

# Rename Zend Studio's socket file to something else

# If you didn't choose the default, installation path then

# use the path to that location instead

mv /usr/local/zend/mysql/tmp/mysql.sock /usr/local/zend/mysql/tmp/mysql_old.sock

# My MySQL installation's socket file is located in /tmp

# If your's is different, then you will have to modify the source path

ln -s /tmp/mysql.sock /usr/local/zend/mysql/tmp

August 2018
M	T	W	T	F	S	S
« Mar
		1	2	3	4	5
6	7	8	9	10	11	12
13	14	15	16	17	18	19
20	21	22	23	24	25	26
27	28	29	30	31