diff --git a/NoahCrawler/pom.xml b/NoahCrawler/pom.xml index 271f2b7..2b26b44 100644 --- a/NoahCrawler/pom.xml +++ b/NoahCrawler/pom.xml @@ -24,4 +24,11 @@ + + + org.jsoup + jsoup + 1.10.1 + + \ No newline at end of file diff --git a/NoahCrawler/src/it/noah/crawler/AbstractNoahCrawler.java b/NoahCrawler/src/it/noah/crawler/AbstractNoahCrawler.java new file mode 100644 index 0000000..13126bc --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/AbstractNoahCrawler.java @@ -0,0 +1,30 @@ +package it.noah.crawler; + +import it.noah.crawler.dom.NoahDOMExplorer; +import it.noah.crawler.dom.tag.Table; +import it.noah.crawler.exception.NoahCrawlerException; +import it.noah.crawler.exception.ObjectNotFoundException; + +import java.io.IOException; + +public abstract class AbstractNoahCrawler implements NoahCrawler { + + private NoahDOMExplorer explorer; + + public AbstractNoahCrawler(String url) { + try { + this.explorer = new NoahDOMExplorer(url); + } catch (IOException e) { + e.printStackTrace(); + } + } + + @Override + public abstract void run() throws NoahCrawlerException; + + public Table getTable(String tableName, boolean header) + throws ObjectNotFoundException { + return explorer.getTable(tableName, header); + } + +} diff --git a/NoahCrawler/src/it/noah/crawler/NoahCrawler.java b/NoahCrawler/src/it/noah/crawler/NoahCrawler.java new file mode 100644 index 0000000..4c4dcd3 --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/NoahCrawler.java @@ -0,0 +1,9 @@ +package it.noah.crawler; + +import it.noah.crawler.exception.NoahCrawlerException; + +public interface NoahCrawler { + + public void run() throws NoahCrawlerException; + +} diff --git a/NoahCrawler/src/it/noah/crawler/annotation/Column.java b/NoahCrawler/src/it/noah/crawler/annotation/Column.java new file mode 100644 index 0000000..3254dd3 --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/annotation/Column.java @@ -0,0 +1,14 @@ +package it.noah.crawler.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.RUNTIME) +public @interface Column { + + public String name(); + +} diff --git a/NoahCrawler/src/it/noah/crawler/annotation/NoahCrawlerModel.java b/NoahCrawler/src/it/noah/crawler/annotation/NoahCrawlerModel.java new file mode 100644 index 0000000..ce444e7 --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/annotation/NoahCrawlerModel.java @@ -0,0 +1,12 @@ +package it.noah.crawler.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.TYPE) +@Retention(RetentionPolicy.RUNTIME) +public @interface NoahCrawlerModel { + +} diff --git a/NoahCrawler/src/it/noah/crawler/annotation/Transiet.java b/NoahCrawler/src/it/noah/crawler/annotation/Transiet.java new file mode 100644 index 0000000..d9bdb67 --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/annotation/Transiet.java @@ -0,0 +1,12 @@ +package it.noah.crawler.annotation; + +import java.lang.annotation.ElementType; +import java.lang.annotation.Retention; +import java.lang.annotation.RetentionPolicy; +import java.lang.annotation.Target; + +@Target(ElementType.FIELD) +@Retention(RetentionPolicy.RUNTIME) +public @interface Transiet { + +} diff --git a/NoahCrawler/src/it/noah/crawler/converter/Converter.java b/NoahCrawler/src/it/noah/crawler/converter/Converter.java new file mode 100644 index 0000000..d9301d4 --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/converter/Converter.java @@ -0,0 +1,11 @@ +package it.noah.crawler.converter; + +import it.noah.crawler.dom.tag.Table; + +import java.util.List; + +public interface Converter { + + public List convertTable(Table table); + +} diff --git a/NoahCrawler/src/it/noah/crawler/converter/impl/ComuneConverter.java b/NoahCrawler/src/it/noah/crawler/converter/impl/ComuneConverter.java new file mode 100644 index 0000000..9f06c39 --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/converter/impl/ComuneConverter.java @@ -0,0 +1,18 @@ +package it.noah.crawler.converter.impl; + +import it.noah.crawler.converter.Converter; +import it.noah.crawler.dom.tag.Table; +import it.noah.crawler.model.Comune; + +import java.util.ArrayList; +import java.util.List; + +public class ComuneConverter implements Converter { + + @Override + public List convertTable(Table table) { + List comuni = new ArrayList(); + return comuni; + } + +} diff --git a/NoahCrawler/src/it/noah/crawler/converter/impl/ProvinciaConverter.java b/NoahCrawler/src/it/noah/crawler/converter/impl/ProvinciaConverter.java new file mode 100644 index 0000000..5e90dbc --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/converter/impl/ProvinciaConverter.java @@ -0,0 +1,26 @@ +package it.noah.crawler.converter.impl; + +import java.util.ArrayList; +import java.util.List; + +import it.noah.crawler.annotation.Column; +import it.noah.crawler.converter.Converter; +import it.noah.crawler.dom.tag.Table; +import it.noah.crawler.model.Provincia; + +public class ProvinciaConverter implements Converter { + + @Override + public List convertTable(Table table) { + List province = new ArrayList(); + int j = 0; + for (int i = 0; i < table.columns(); i++) { + Provincia provincia = new Provincia(); + Column annotations = provincia.getClass() + .getAnnotation(Column.class); + province.add(provincia); + } + return province; + } + +} diff --git a/NoahCrawler/src/it/noah/crawler/dom/NoahDOMExplorer.java b/NoahCrawler/src/it/noah/crawler/dom/NoahDOMExplorer.java new file mode 100644 index 0000000..70a77eb --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/dom/NoahDOMExplorer.java @@ -0,0 +1,51 @@ +package it.noah.crawler.dom; + +import it.noah.crawler.dom.tag.Table; +import it.noah.crawler.dom.tag.builder.TableBuilder; +import it.noah.crawler.exception.ObjectNotFoundException; + +import java.io.IOException; + +import org.jsoup.Jsoup; +import org.jsoup.nodes.Document; +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +public class NoahDOMExplorer { + + private Document document; + + public NoahDOMExplorer(String url) throws IOException { + document = Jsoup.connect(url).get(); + } + + public synchronized Document getDocument() { + return document; + } + + public synchronized Table getTable(String tableName, boolean header) + throws ObjectNotFoundException { + if (document == null) { + throw new ObjectNotFoundException( + "Documento non presente in sessione"); + } + Elements elements = document.getElementsByTag("table"); + Element element = selectByName(elements, tableName); + TableBuilder tableBuilder = new TableBuilder(); + return tableBuilder.build(element, header); + } + + private synchronized Element selectByName(Elements elements, String name) { + Element selected = null; + if (name == null || elements == null) { + throw new IllegalArgumentException("Parametri non validi!!!"); + } + for (Element current : elements) { + if (name.equals(current.className())) { + selected = current; + } + } + return selected; + } + +} diff --git a/NoahCrawler/src/it/noah/crawler/dom/tag/Cell.java b/NoahCrawler/src/it/noah/crawler/dom/tag/Cell.java new file mode 100644 index 0000000..245ee45 --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/dom/tag/Cell.java @@ -0,0 +1,14 @@ +package it.noah.crawler.dom.tag; + +public class Cell { + private String value; + + public void setValue(String value) { + this.value = value; + } + + public String getValue() { + return value; + } + +} diff --git a/NoahCrawler/src/it/noah/crawler/dom/tag/Table.java b/NoahCrawler/src/it/noah/crawler/dom/tag/Table.java new file mode 100644 index 0000000..20e1654 --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/dom/tag/Table.java @@ -0,0 +1,33 @@ +package it.noah.crawler.dom.tag; + +import java.util.List; + +public class Table { + private TableHeader header; + private List rows; + + public Table() { + header = new TableHeader(); + } + + public void setHeader(TableHeader header) { + this.header = header; + } + + public TableHeader getHeader() { + return header; + } + + public void setRows(List rows) { + this.rows = rows; + } + + public List getRows() { + return rows; + } + + public int columns() { + return header.size(); + } + +} diff --git a/NoahCrawler/src/it/noah/crawler/dom/tag/TableHeader.java b/NoahCrawler/src/it/noah/crawler/dom/tag/TableHeader.java new file mode 100644 index 0000000..fbdb662 --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/dom/tag/TableHeader.java @@ -0,0 +1,25 @@ +package it.noah.crawler.dom.tag; + +import java.util.LinkedList; +import java.util.List; + +public class TableHeader { + private List cells = new LinkedList(); + + public void addColumn(Cell cell) { + cells.add(cell); + } + + public List getColumns() { + return cells; + } + + public Cell getColumn(int index) { + return cells.get(index); + } + + public int size() { + return cells.size(); + } + +} diff --git a/NoahCrawler/src/it/noah/crawler/dom/tag/TableRow.java b/NoahCrawler/src/it/noah/crawler/dom/tag/TableRow.java new file mode 100644 index 0000000..095487c --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/dom/tag/TableRow.java @@ -0,0 +1,21 @@ +package it.noah.crawler.dom.tag; + +import java.util.LinkedList; +import java.util.List; + +public class TableRow { + private List cells = new LinkedList(); + + public List getRow() { + return cells; + } + + public void addRow(List row) { + cells = row; + } + + public int size() { + return cells.size(); + } + +} diff --git a/NoahCrawler/src/it/noah/crawler/dom/tag/builder/TableBuilder.java b/NoahCrawler/src/it/noah/crawler/dom/tag/builder/TableBuilder.java new file mode 100644 index 0000000..dd4e7c7 --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/dom/tag/builder/TableBuilder.java @@ -0,0 +1,26 @@ +package it.noah.crawler.dom.tag.builder; + +import it.noah.crawler.dom.tag.Table; + +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +public class TableBuilder { + + public Table build(Element element, boolean headerFlag) { + Table table = new Table(); + Elements elements = element.select("tr"); + Element header; + if (headerFlag) { + header = elements.select("tr").first(); + header.remove(); + TableHeaderBuilder thb = new TableHeaderBuilder(); + table.setHeader(thb.build(header)); + } + Elements rows = elements.select("tr"); + TableRowBuilder thr = new TableRowBuilder(); + // table.setBody(thr.build(rows)); TODO + return table; + } + +} diff --git a/NoahCrawler/src/it/noah/crawler/dom/tag/builder/TableHeaderBuilder.java b/NoahCrawler/src/it/noah/crawler/dom/tag/builder/TableHeaderBuilder.java new file mode 100644 index 0000000..ddf11ca --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/dom/tag/builder/TableHeaderBuilder.java @@ -0,0 +1,12 @@ +package it.noah.crawler.dom.tag.builder; + +import it.noah.crawler.dom.tag.TableHeader; + +import org.jsoup.nodes.Element; + +public class TableHeaderBuilder { + + public TableHeader build(Element element) { + return null; + } +} diff --git a/NoahCrawler/src/it/noah/crawler/dom/tag/builder/TableRowBuilder.java b/NoahCrawler/src/it/noah/crawler/dom/tag/builder/TableRowBuilder.java new file mode 100644 index 0000000..4054f4b --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/dom/tag/builder/TableRowBuilder.java @@ -0,0 +1,22 @@ +package it.noah.crawler.dom.tag.builder; + +import it.noah.crawler.dom.tag.Cell; +import it.noah.crawler.dom.tag.TableRow; + +import org.jsoup.nodes.Element; +import org.jsoup.select.Elements; + +public class TableRowBuilder { + + public TableRow build(Elements rows) { + TableRow tableRow = new TableRow(); + for (Element row : rows) { + Elements tds = row.select("td"); + Cell cell = new Cell(); + cell.setValue(tds.text()); + // tableRow.addColumn(cell); TODO + } + return tableRow; + } + +} diff --git a/NoahCrawler/src/it/noah/crawler/enums/CrawlerEnum.java b/NoahCrawler/src/it/noah/crawler/enums/CrawlerEnum.java new file mode 100644 index 0000000..a01a5c8 --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/enums/CrawlerEnum.java @@ -0,0 +1,5 @@ +package it.noah.crawler.enums; + +public enum CrawlerEnum { + PROVINCIA_CRAWLER, COMUNE_CRAWLER +} diff --git a/NoahCrawler/src/it/noah/crawler/enums/UrlEnum.java b/NoahCrawler/src/it/noah/crawler/enums/UrlEnum.java new file mode 100644 index 0000000..ef4bf8a --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/enums/UrlEnum.java @@ -0,0 +1,17 @@ +package it.noah.crawler.enums; + +public enum UrlEnum { + PROVINCE_URL( + "http://www.aci.it/i-servizi/normative/codice-della-strada/elenco-sigle-province-ditalia.html"), + COMUNI_URL("https://it.wikipedia.org/wiki/Citt%C3%A0_d'Italia"); + + private String url; + + UrlEnum(String url) { + this.url = url; + } + + public String getUrl() { + return this.url; + } +} diff --git a/NoahCrawler/src/it/noah/crawler/exception/NoahCrawlerException.java b/NoahCrawler/src/it/noah/crawler/exception/NoahCrawlerException.java new file mode 100644 index 0000000..26fe40f --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/exception/NoahCrawlerException.java @@ -0,0 +1,18 @@ +package it.noah.crawler.exception; + +public class NoahCrawlerException extends Exception { + + private static final long serialVersionUID = 1L; + + private String message; + + public NoahCrawlerException(String message) { + this.message = message; + } + + @Override + public String getMessage() { + return this.message; + } + +} diff --git a/NoahCrawler/src/it/noah/crawler/exception/ObjectNotFoundException.java b/NoahCrawler/src/it/noah/crawler/exception/ObjectNotFoundException.java new file mode 100644 index 0000000..008afd6 --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/exception/ObjectNotFoundException.java @@ -0,0 +1,18 @@ +package it.noah.crawler.exception; + +public class ObjectNotFoundException extends Exception { + + private static final long serialVersionUID = -4004459574648431770L; + + private String message; + + public ObjectNotFoundException(String message) { + this.message = message; + } + + @Override + public String getMessage() { + return this.message; + } + +} diff --git a/NoahCrawler/src/it/noah/crawler/factory/NoahCrawlerFactory.java b/NoahCrawler/src/it/noah/crawler/factory/NoahCrawlerFactory.java new file mode 100644 index 0000000..9ee7c06 --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/factory/NoahCrawlerFactory.java @@ -0,0 +1,21 @@ +package it.noah.crawler.factory; + +import it.noah.crawler.NoahCrawler; +import it.noah.crawler.enums.CrawlerEnum; +import it.noah.crawler.impl.ComuneNoahCrawler; +import it.noah.crawler.impl.ProvinciaNoahCrawler; + +import java.io.IOException; + +public class NoahCrawlerFactory { + + public static NoahCrawler getInstance(CrawlerEnum crawler) throws IOException { + if (crawler == CrawlerEnum.PROVINCIA_CRAWLER) { + return new ProvinciaNoahCrawler(); + } else if (crawler == CrawlerEnum.COMUNE_CRAWLER) { + return new ComuneNoahCrawler(); + } + return null; + } + +} diff --git a/NoahCrawler/src/it/noah/crawler/impl/ComuneNoahCrawler.java b/NoahCrawler/src/it/noah/crawler/impl/ComuneNoahCrawler.java new file mode 100644 index 0000000..2932262 --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/impl/ComuneNoahCrawler.java @@ -0,0 +1,36 @@ +package it.noah.crawler.impl; + +import it.noah.crawler.AbstractNoahCrawler; +import it.noah.crawler.NoahCrawler; +import it.noah.crawler.converter.impl.ComuneConverter; +import it.noah.crawler.dom.tag.Table; +import it.noah.crawler.enums.UrlEnum; +import it.noah.crawler.exception.NoahCrawlerException; +import it.noah.crawler.exception.ObjectNotFoundException; +import it.noah.crawler.model.Comune; + +import java.io.IOException; +import java.util.List; + +public class ComuneNoahCrawler extends AbstractNoahCrawler implements + NoahCrawler { + + public ComuneNoahCrawler() throws IOException { + super(UrlEnum.COMUNI_URL.getUrl()); + } + + @Override + public void run() throws NoahCrawlerException { + try { + List comuni = convertTable2Comuni(getTable("", true)); + // TODO una volta creata la lista va salvata sul db + } catch (ObjectNotFoundException e) { + e.printStackTrace(); + } + } + + private List convertTable2Comuni(Table table) { + return new ComuneConverter().convertTable(table); + } + +} diff --git a/NoahCrawler/src/it/noah/crawler/impl/ProvinciaNoahCrawler.java b/NoahCrawler/src/it/noah/crawler/impl/ProvinciaNoahCrawler.java new file mode 100644 index 0000000..5e64a9b --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/impl/ProvinciaNoahCrawler.java @@ -0,0 +1,36 @@ +package it.noah.crawler.impl; + +import it.noah.crawler.AbstractNoahCrawler; +import it.noah.crawler.NoahCrawler; +import it.noah.crawler.converter.impl.ProvinciaConverter; +import it.noah.crawler.dom.tag.Table; +import it.noah.crawler.enums.UrlEnum; +import it.noah.crawler.exception.NoahCrawlerException; +import it.noah.crawler.exception.ObjectNotFoundException; +import it.noah.crawler.model.Provincia; + +import java.io.IOException; +import java.util.List; + +public class ProvinciaNoahCrawler extends AbstractNoahCrawler implements + NoahCrawler { + + public ProvinciaNoahCrawler() throws IOException { + super(UrlEnum.PROVINCE_URL.getUrl()); + } + + @Override + public void run() throws NoahCrawlerException { + try { + List province = convertTable2Provincia(getTable("", true)); + // TODO una volta creata la lista va salvata sul db + } catch (ObjectNotFoundException e) { + e.printStackTrace(); + } + } + + private List convertTable2Provincia(Table table) { + return new ProvinciaConverter().convertTable(table); + } + +} diff --git a/NoahCrawler/src/it/noah/crawler/model/Comune.java b/NoahCrawler/src/it/noah/crawler/model/Comune.java new file mode 100644 index 0000000..d5b6d80 --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/model/Comune.java @@ -0,0 +1,65 @@ +package it.noah.crawler.model; + +import it.noah.crawler.annotation.Column; +import it.noah.crawler.annotation.NoahCrawlerModel; +import it.noah.crawler.annotation.Transiet; + +@NoahCrawlerModel +public class Comune { + + @Transiet + private Long id; + + @Column(name = "") + private String nome; + + @Column(name = "") + private String cap; + + @Column(name = "") + private Provincia provincia; + + @Column(name = "") + private String regione; + + public void setId(Long id) { + this.id = id; + } + + public Long getId() { + return id; + } + + public String getNome() { + return nome; + } + + public void setNome(String nome) { + this.nome = nome; + } + + public String getCap() { + return cap; + } + + public void setCap(String cap) { + this.cap = cap; + } + + public Provincia getProvincia() { + return provincia; + } + + public void setProvincia(Provincia provincia) { + this.provincia = provincia; + } + + public void setRegione(String regione) { + this.regione = regione; + } + + public String getRegione() { + return regione; + } + +} diff --git a/NoahCrawler/src/it/noah/crawler/model/Provincia.java b/NoahCrawler/src/it/noah/crawler/model/Provincia.java new file mode 100644 index 0000000..a447d53 --- /dev/null +++ b/NoahCrawler/src/it/noah/crawler/model/Provincia.java @@ -0,0 +1,67 @@ +package it.noah.crawler.model; + +import it.noah.crawler.annotation.Column; +import it.noah.crawler.annotation.NoahCrawlerModel; +import it.noah.crawler.annotation.Transiet; + +import java.util.List; + +@NoahCrawlerModel +public class Provincia { + + @Transiet + private Long id; + + @Column(name = "Provincia") + private String nome; + + @Column(name = "Sigla") + private String sigla; + + @Column(name = "Regione") + private String regione; + + @Transiet + private List comuni; + + public void setId(Long id) { + this.id = id; + } + + public Long getId() { + return id; + } + + public String getNome() { + return nome; + } + + public void setNome(String nome) { + this.nome = nome; + } + + public String getSigla() { + return sigla; + } + + public void setSigla(String sigla) { + this.sigla = sigla; + } + + public void setComuni(List comuni) { + this.comuni = comuni; + } + + public List getComuni() { + return comuni; + } + + public void setRegione(String regione) { + this.regione = regione; + } + + public String getRegione() { + return regione; + } + +}